From 501294bf5a496ceeb1bb51e6e96c1e2a935e23ab Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 20 Jul 2022 21:36:22 +0200 Subject: [PATCH 01/75] fix in GA action --- .github/workflows/python-frog.yml | 2 +- build-wheels.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-frog.yml b/.github/workflows/python-frog.yml index 2b42f1f..ae9919d 100644 --- a/.github/workflows/python-frog.yml +++ b/.github/workflows/python-frog.yml @@ -136,7 +136,7 @@ jobs: CXX: ${{ matrix.compiler }} run: | git clone https://github.com/LanguageMachines/frog; - cd frogdata; + cd frog; bash bootstrap.sh; ./configure; make; diff --git a/build-wheels.sh b/build-wheels.sh index a9fe7e8..be35cd5 100755 --- a/build-wheels.sh +++ b/build-wheels.sh @@ -14,7 +14,6 @@ SYSTEM_PACKAGES=$3 PRE_BUILD_COMMAND=$4 PACKAGE_PATH=$5 PIP_WHEEL_ARGS=$6 -BUILD_DEPS_OPTS=$7 # Temporary workaround for LD_LIBRARY_PATH issue. See # https://github.com/RalfG/python-wheels-manylinux-build/issues/26 From f6b54008fa2ac72acec67db896a49a04ad76dacf Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 20 Jul 2022 21:43:11 +0200 Subject: [PATCH 02/75] comment update --- build-deps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-deps.sh b/build-deps.sh index 56b2d20..ce663e5 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -1,7 +1,8 @@ #!/bin/sh # Builds dependencies (latest stable releases) from source -# Used for building wheels +# Used for building wheels. Invoke via 'make wheels' rather +# than directly! set -e From 9ae63579a860855c28524efad774d495309b3cb5 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 21 Jul 2022 11:49:21 +0200 Subject: [PATCH 03/75] reference latest uctodata release --- frog_wrapper.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index 50db6d7..d766705 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -21,7 +21,7 @@ cimport libfolia_classes cimport frog_classes FROGDATAVERSION = "0.20" -UCTODATAVERSION = "0.8" +UCTODATAVERSION = "0.9.1" try: from folia.main import Document as FoliaPyDocument From 15751da9f230f08ac8c1b093f53a2a40ff9387fa Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Fri, 22 Jul 2022 12:47:24 +0200 Subject: [PATCH 04/75] updated FROGDATAVERSION to 0.21 --- frog_wrapper.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index d766705..c5340ad 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -20,7 +20,7 @@ import sys cimport libfolia_classes cimport frog_classes -FROGDATAVERSION = "0.20" +FROGDATAVERSION = "0.21" UCTODATAVERSION = "0.9.1" try: From 871129515a1e7aa168c84ef40cb54aab76c1d75e Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Fri, 22 Jul 2022 13:54:13 +0200 Subject: [PATCH 05/75] build-deps: use temporary work dir --- build-deps.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build-deps.sh b/build-deps.sh index ce663e5..32b7f4f 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -32,6 +32,9 @@ if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then fi [ -z "$PREFIX" ] && PREFIX="/usr/local/" +PWD="$(pwd)" +BUILDDIR="$(mktemp -dt "build-deps.XXXXXX")" +cd "$BUILDDIR" for PACKAGE in tklauser/libtar LanguageMachines/ticcutils LanguageMachines/libfolia LanguageMachines/uctodata LanguageMachines/ucto LanguageMachines/timbl LanguageMachines/mbt LanguageMachines/frogdata LanguageMachines/frog; do echo "Git cloning $PACKAGE ">&2 git clone https://github.com/$PACKAGE @@ -57,5 +60,7 @@ for PACKAGE in tklauser/libtar LanguageMachines/ticcutils LanguageMachines/libfo make install cd .. done +cd $PWD +[ -n "$BUILDDIR" ] && rm -Rf "$BUILDDIR" echo "Dependencies installed" >&2 From 8851bc407059ef30b67e6bfda20677cf9bf5aa38 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Fri, 22 Jul 2022 21:41:10 +0200 Subject: [PATCH 06/75] allow selecting another tag when building wheels --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index b8cd3d8..b8f42cb 100644 --- a/Makefile +++ b/Makefile @@ -5,17 +5,18 @@ SYSTEM_PACKAGES=libicu-devel libxml2-devel libxslt-devel libexttextcat zlib-deve PRE_BUILD_COMMAND=/io/build-deps.sh PACKAGE_PATH= PIP_WHEEL_ARGS=-w ./dist --no-deps +TAG=latest .PHONY: wheels wheels: - docker pull quay.io/pypa/manylinux_2_28_x86_64 - docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64 /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES}" "${PRE_BUILD_COMMAND}" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" + docker pull quay.io/pypa/manylinux_2_28_x86_64:${TAG} + docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES}" "${PRE_BUILD_COMMAND}" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" .PHONY: devwheels devwheels: #builds against latest development versions of everything, these wheels should NOT be published! - docker pull quay.io/pypa/manylinux_2_28_x86_64 - docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64 /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES}" "${PRE_BUILD_COMMAND} --devel" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" + docker pull quay.io/pypa/manylinux_2_28_x86_64:${TAG} + docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES}" "${PRE_BUILD_COMMAND} --devel" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" echo "Built against development versions, DO NOT PUBLISH these wheels!" .PHONY: install From 4a384f54094f62920ecf365c67b990482e1d38fc Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 3 Nov 2022 13:29:02 +0100 Subject: [PATCH 07/75] added support for musl to build-wheels --- Makefile | 15 +++++++++++---- build-wheels.sh | 9 ++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b8f42cb..c80b4e7 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,29 @@ PY_VERSIONS=cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 BUILD_REQUIREMENTS=cython #system packages (yum, manylinux_2_28 uses AlmaLinux) -SYSTEM_PACKAGES=libicu-devel libxml2-devel libxslt-devel libexttextcat zlib-devel bzip2-devel libtool autoconf-archive autoconf automake m4 wget +SYSTEM_PACKAGES_RHEL=libicu-devel libxml2-devel libxslt-devel libexttextcat zlib-devel bzip2-devel libtool autoconf-archive autoconf automake m4 wget +#musllinux_1_1_x86_64 builds on Alpine v3.12 +SYSTEM_PACKAGES_ALPINE=icu-dev libxml2-dev libxslt-dev libexttextcat-dev zlib-dev bzip2-dev libtool autoconf-archive autoconf automake m4 wget PRE_BUILD_COMMAND=/io/build-deps.sh PACKAGE_PATH= PIP_WHEEL_ARGS=-w ./dist --no-deps TAG=latest .PHONY: wheels -wheels: +wheels: muslwheels docker pull quay.io/pypa/manylinux_2_28_x86_64:${TAG} - docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES}" "${PRE_BUILD_COMMAND}" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" + docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES_RHEL}" "${PRE_BUILD_COMMAND}" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" + +.PHONY: muslwheels +muslwheels: + docker pull quay.io/pypa/musllinux_1_1_x86_64:${TAG} + docker run --rm -e PLAT=musllinux_1_1_x86_64 -v `pwd`:/io quay.io/pypa/musllinux_1_1_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES_ALPINE}" "${PRE_BUILD_COMMAND}" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" .PHONY: devwheels devwheels: #builds against latest development versions of everything, these wheels should NOT be published! docker pull quay.io/pypa/manylinux_2_28_x86_64:${TAG} - docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES}" "${PRE_BUILD_COMMAND} --devel" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" + docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES_RHEL}" "${PRE_BUILD_COMMAND} --devel" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" echo "Built against development versions, DO NOT PUBLISH these wheels!" .PHONY: install diff --git a/build-wheels.sh b/build-wheels.sh index be35cd5..d086eb8 100755 --- a/build-wheels.sh +++ b/build-wheels.sh @@ -27,6 +27,9 @@ if [ -n "$SYSTEM_PACKAGES" ]; then apt-get install -y $SYSTEM_PACKAGES || { echo "Installing apt package(s) failed."; exit 1; } elif command -v yum >/dev/null; then yum install -y $SYSTEM_PACKAGES || { echo "Installing yum package(s) failed."; exit 1; } + elif command -v apk >/dev/null; then + apk add $SYSTEM_PACKAGES rsync || { echo "Installing apk package(s) failed."; exit 1; } + rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ #the image has it's own aclocal installation, copy autoconf-archive files that do not exist yet (and hope for the best) else echo "Package managers apt or yum not found."; exit 1; fi @@ -68,4 +71,8 @@ if [[ -f "$failed_wheels" ]]; then fi echo "Succesfully build wheels:" -find . -type f -iname "*-manylinux*.whl" +if command -v apk >/dev/null; then + find . -type f -iname "*-musllinux*.whl" +else + find . -type f -iname "*-manylinux*.whl" +fi From 4d6294a59952ca3771aa87289f532a9f3a539256 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 3 Nov 2022 13:46:20 +0100 Subject: [PATCH 08/75] added python 3.11 support for wheels --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c80b4e7..1bd0102 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PY_VERSIONS=cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 +PY_VERSIONS=cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 BUILD_REQUIREMENTS=cython #system packages (yum, manylinux_2_28 uses AlmaLinux) SYSTEM_PACKAGES_RHEL=libicu-devel libxml2-devel libxslt-devel libexttextcat zlib-devel bzip2-devel libtool autoconf-archive autoconf automake m4 wget From 8dfc4a9850164816ca92e3e28cc7cbe0f41fe1ec Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 29 Nov 2022 17:43:06 +0100 Subject: [PATCH 09/75] added container support --- Dockerfile | 12 ++++++++++++ README.rst | 12 +++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..75e328f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM proycon/frog +LABEL org.opencontainers.image.authors="Maarten van Gompel " +LABEL description="Frog - A Tagger-Lemmatizer-Morphological-Analyzer-Dependency-Parser for Dutch, container image with the Python binding" + + +COPY . /usr/src/python-frog +RUN BUILD_PACKAGES="build-base libtool libtar-dev bzip2-dev icu-dev libxml2-dev libexttextcat-dev python3-dev" &&\ + mkdir -p /usr/src/python-frog &&\ + apk add python3 py3-wheel py3-pip cython $BUILD_PACKAGES &&\ + cd /usr/src/python-frog && pip install . && apk del $BUILD_PACKAGES && rm -Rf /usr/src/python-frog + +ENTRYPOINT [ "python3" ] diff --git a/README.rst b/README.rst index 82446a6..dd72ed7 100644 --- a/README.rst +++ b/README.rst @@ -66,7 +66,17 @@ Then install this binding using ``pip`` (preferably in a Python virtual environm Windows ~~~~~~~~~~ -Not supported natively, but you should be able to use the Frog python binding if you use WSL. +Not supported natively, but you should be able to use the Frog python binding if you use WSL, or using Docker containers (see below). + +Docker/OCI Containers +~~~~~~~~~~~~~~~~~~~~~~~ + +A Docker/OCI container image is available containing Python, frog, and python-frog:: + + docker pull proycon/python-frog + docker run -t -i proycon/python-frog + +You can also build the container from scratch from this repository with the included `Dockerfile`. Usage ------------------ From 9d7dcdfdbe7353bea088c59ab3d14f2b9cd34b00 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 30 Nov 2022 14:29:44 +0100 Subject: [PATCH 10/75] added missing frog_wrapper.pyx for source distribution --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a305093..68ca593 100755 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ cmdclass = {'build_ext': build_ext}, requires = ['frog (>=0.25)'], install_requires=['Cython'], + data_files = [("sources",["frog_wrapper.pyx"])], classifiers=[ "Development Status :: 5 - Production/Stable", "Topic :: Text Processing :: Linguistic", From 3e579e0261778a8936bab3f6f520fb856b27a2e2 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 30 Nov 2022 14:30:31 +0100 Subject: [PATCH 11/75] added attempted support for mac arm64 --- setup.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 68ca593..29b469d 100755 --- a/setup.py +++ b/setup.py @@ -9,12 +9,22 @@ includedirs = [] libdirs = [] if platform.system() == "Darwin": - #we are running on Mac OS X with homebrew, stuff is in specific locations: - libdirs.append("/usr/local/opt/icu4c/lib") - includedirs.append("/usr/local/opt/icu4c/include") - libdirs.append("/usr/local/opt/libxml2/lib") - includedirs.append("/usr/local/opt/libxml2/include") - includedirs.append("/usr/local/opt/libxml2/include/libxml2") + #we are running on Mac OS X (with homebrew hopefully), stuff is in specific locations: + if platform.machine().lower() == "arm64": + libdirs.append("/opt/homebrew/lib") + includedirs.append("/opt/homebrew/include") + libdirs.append("/opt/homebrew/icu4c/lib") + includedirs.append("/opt/homebrew/icu4c/include") + libdirs.append("/opt/homebrew/libxml2/lib") + includedirs.append("/opt/homebrew/libxml2/include") + includedirs.append("/opt/homebrew/libxml2/include/libxml2") + else: + #we are running on Mac OS X with homebrew, stuff is in specific locations: + libdirs.append("/usr/local/opt/icu4c/lib") + includedirs.append("/usr/local/opt/icu4c/include") + libdirs.append("/usr/local/opt/libxml2/lib") + includedirs.append("/usr/local/opt/libxml2/include") + includedirs.append("/usr/local/opt/libxml2/include/libxml2") #add some common default paths includedirs += ['/usr/include/', '/usr/include/libxml2','/usr/local/include/' ] @@ -47,7 +57,7 @@ setup( name = 'python-frog', - version = '0.6.0', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.1', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to FROG, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 9070080b8d0a0ce4f1cd7e7d6d9d0cc1b4ffcd43 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 30 Nov 2022 14:34:19 +0100 Subject: [PATCH 12/75] minor Makefile update --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1bd0102..a7bee70 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,10 @@ PIP_WHEEL_ARGS=-w ./dist --no-deps TAG=latest .PHONY: wheels -wheels: muslwheels +wheels: manywheels muslwheels + +.PHONY: manywheels +manywheels: docker pull quay.io/pypa/manylinux_2_28_x86_64:${TAG} docker run --rm -e PLAT=manylinux_2_28_x86_64 -v `pwd`:/io quay.io/pypa/manylinux_2_28_x86_64:${TAG} /io/build-wheels.sh "${PY_VERSIONS}" "${BUILD_REQUIREMENTS}" "${SYSTEM_PACKAGES_RHEL}" "${PRE_BUILD_COMMAND}" "${PACKAGE_PATH}" "${PIP_WHEEL_ARGS}" From f7d737ab7697434751cf0a6032c7e777b57fbe59 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 30 Nov 2022 18:00:46 +0100 Subject: [PATCH 13/75] ci: added wheel building using Github Actions --- .github/workflows/wheels.yml | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/wheels.yml diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..0e1b179 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,47 @@ +#Note: This script does not used build-wheels.sh nor the Makefile included in this repo (that's for local usage) + +name: Build Wheels + +on: [workflow_dispatch] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, macOS-11] + + steps: + - uses: actions/checkout@v3 + + # Used to host cibuildwheel + - uses: actions/setup-python@v3 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS_LINUX: "x86_64" + CIBW_ARCHS_MACOS: "x86_64 arm64" + CIBW_BEFORE_ALL_LINUX: > + if command -v apt-get; then + apt-get -y git libicu-dev libxml2-dev libexttextcat-dev libxslt1-dev libbz2-dev zlib1g-dev autoconf automake autoconf-archive libtool autotools-dev gcc g++ make + elif command -v yum; then + yum install -y git libicu-devel libxml2-devel libxslt-devel libexttextcat zlib-devel bzip2-devel libtool autoconf-archive autoconf automake m4 wget + elif command -v apk; then + apk add build-base git autoconf-archive autoconf automake libtool bzip2-dev icu-dev libxml2-dev libexttextcat-dev libtool rsync && + rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ + fi && + ./build-deps.sh + CIBW_BEFORE_ALL_MACOS: > + brew tap fbkarsdorp/homebrew-lamachine && + brew install frog + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + CIBW_SKIP: "*-win* *-manylinux_i686 pp*" + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl From 63afaecbf0c02a3c61476b56330b59ecf2d45f29 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 30 Nov 2022 18:18:36 +0100 Subject: [PATCH 14/75] ci: added pyproject.toml with build dependencies (PEP518) --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fb4a3f0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "Cython" +] + +build-backend = "setuptools.build_meta" From af3f5131cb0fd48ad5e660d73b8975d32e16a095 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 1 Dec 2022 13:52:52 +0100 Subject: [PATCH 15/75] README update --- README.rst | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/README.rst b/README.rst index dd72ed7..3b4b6a8 100644 --- a/README.rst +++ b/README.rst @@ -20,14 +20,11 @@ dependency parsing. The tool itself is implemented in C++ Installation ---------------- -Linux -~~~~~~~~~~ - -On modern Linux distributions, we recommend you use a Python virtual environment and install using ``pip``:: +We recommend you use a Python virtual environment and install using ``pip``:: pip install python-frog -When possible on your system (glibc >= 2.28, x86_64), this will install the binary +When possible on your system, this will install the binary Python wheels *that include Frog and all necessary dependencies* **except for** frogdata. To download and install the data (in ``~/.config/frog``) you then only need to run the following once:: @@ -38,35 +35,21 @@ If you want language detection support, ensure you the have `libexttextcat` package (if provided by your distribution) installed prior to executing the above command. -If the binary wheels are not available for your distribution, you will need to first install Frog yourself. - -Alpine Linux -~~~~~~~~~~~~~~ - -First install Cython and Frog (``apk add cython frog frog-dev``), then ``pip install python-frog`` (preferably in a Python virtual environment) - -Arch Linux -~~~~~~~~~~~~~~ - -You can alternatively use the `AUR package `_ . +If the binary wheels are not available for your system, you will need to first +install `Frog `_ yourself and then +run ``pip install python-frog`` to install this python binding, it will then be +compiled from source. The following instructions apply in that case: +On Arch Linux, you can alternatively use the `AUR package `_ . -macOS -~~~~~~~~ - -First use `homebrew `_ to install `Frog `_:: +On macOS; first use `homebrew `_ to install `Frog `_:: brew tap fbkarsdorp/homebrew-lamachine - brew install frog - -Then install this binding using ``pip`` (preferably in a Python virtual environment):: - - pip install python-frog + brew install ucto -Windows -~~~~~~~~~~ +On Alpine Linux, run: ``apk add cython frog frog-dev`` -Not supported natively, but you should be able to use the Frog python binding if you use WSL, or using Docker containers (see below). +Windows is not supported natively at all, but you should be able to use the Ucto python binding if you use WSL, or using Docker containers (see below). Docker/OCI Containers ~~~~~~~~~~~~~~~~~~~~~~~ From 837e9dc3db0b5560b9a2fc17877b4750abe5b5d1 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 21 Dec 2022 13:31:49 +0100 Subject: [PATCH 16/75] doc: adapted example so 'frog' variable isn't shadowed In response to question by @roelsmeets --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 3b4b6a8..d22a8c1 100644 --- a/README.rst +++ b/README.rst @@ -68,9 +68,9 @@ Example: .. code:: python - import frog + from frog import Frog, FrogOptions - frog = frog.Frog(frog.FrogOptions(parser=False)) + frog = Frog(FrogOptions(parser=False)) output = frog.process_raw("Dit is een test") print("RAW OUTPUT=",output) output = frog.process("Dit is nog een test.") @@ -124,14 +124,14 @@ used: .. code:: python - frog = frog.Frog(frog.FrogOptions(parser=False), "/path/to/your/frog.cfg") + frog = Frog(FrogOptions(parser=False), "/path/to/your/frog.cfg") A third parameter, a dictionary, can be used to override specific configuration values (same syntax as Frog's ``--override`` option), you may want to leave the second parameter empty if you want to load the default configuration: .. code:: python - frog = frog.Frog(frog.FrogOptions(parser=False), "", { "tokenizer.rulesFile": "tokconfig-nld-twitter" }) + frog = Frog(FrogOptions(parser=False), "", { "tokenizer.rulesFile": "tokconfig-nld-twitter" }) FoLiA support ------------------ From 88b125c24ebff9e5675038b166fcdafbe5fc4cac Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 3 Jan 2023 13:11:17 +0100 Subject: [PATCH 17/75] dependency update: frog >= 0.26 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 29b469d..3b719f4 100755 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name = 'python-frog', - version = '0.6.1', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.2', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to FROG, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), @@ -66,7 +66,7 @@ url = "https://github.com/proycon/python-frog", ext_modules = extensions, cmdclass = {'build_ext': build_ext}, - requires = ['frog (>=0.25)'], + requires = ['frog (>=0.26)'], install_requires=['Cython'], data_files = [("sources",["frog_wrapper.pyx"])], classifiers=[ From 0b386d277d877b1c714685af818ace0aead36cbc Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 18 Jan 2023 19:16:07 +0100 Subject: [PATCH 18/75] wheels: fixed build, downgraded to manylinux2014 (needed for ubuntu 20.04 compat) --- .github/workflows/wheels.yml | 2 +- build-deps.sh | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 0e1b179..e96b262 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -39,7 +39,7 @@ jobs: CIBW_BEFORE_ALL_MACOS: > brew tap fbkarsdorp/homebrew-lamachine && brew install frog - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - uses: actions/upload-artifact@v3 diff --git a/build-deps.sh b/build-deps.sh index 32b7f4f..d54db99 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -26,9 +26,24 @@ if [ "$ID" = "almalinux" ] && [ -d /usr/local/share/aclocal ]; then export ACLOCAL_PATH=/usr/share/aclocal fi if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then - #they forgot to package libexttextcat-devel? grab one manually: - wget https://github.com/proycon/LaMachine/raw/master/deps/centos8/libexttextcat-devel-3.4.5-2.el8.x86_64.rpm - yum install -y libexttextcat-devel-3.4.5-2.el8.x86_64.rpm + if [ "$VERSION_ID" = "7" ]; then + yum install -y libexttextcat-devel + if [ -d /opt/rh/devtoolset-10/root/usr/lib ]; then + #we are running in the manylinux2014 image + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/opt/rh/devtoolset-10/root/usr/lib + #libxml2 is out of date, compile and install a new one + yum install -y xz + wget https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz + unxz libxml2-2.9.14.tar.xz + tar -xf libxml2-2.9.14.tar + cd libxml2-2.9.14 && ./configure --prefix=/usr/ --without-python && make && make install + cd .. + fi + elif [ "$VERSION_ID" = "8" ]; then + #they forgot to package libexttextcat-devel? grab one manually: + wget https://github.com/proycon/LaMachine/raw/master/deps/centos8/libexttextcat-devel-3.4.5-2.el8.x86_64.rpm + yum install -y libexttextcat-devel-3.4.5-2.el8.x86_64.rpm + fi fi [ -z "$PREFIX" ] && PREFIX="/usr/local/" From 5eef0cf32bec547228f7e728e4babeae82c94eda Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 18 Jan 2023 19:29:25 +0100 Subject: [PATCH 19/75] build-deps fix for CentOS 7 + trailing whitespace fix --- build-deps.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/build-deps.sh b/build-deps.sh index d54db99..0403917 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -21,24 +21,24 @@ get_latest_version() { fi } -if [ "$ID" = "almalinux" ] && [ -d /usr/local/share/aclocal ]; then - #needed for manylinux_2_28 container which ships custom autoconf - export ACLOCAL_PATH=/usr/share/aclocal -fi if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then + if [ -d /usr/local/share/aclocal ]; then + #needed for manylinux_2_28 container which ships custom autoconf, possibly others too? + export ACLOCAL_PATH=/usr/share/aclocal + fi if [ "$VERSION_ID" = "7" ]; then yum install -y libexttextcat-devel - if [ -d /opt/rh/devtoolset-10/root/usr/lib ]; then - #we are running in the manylinux2014 image - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/opt/rh/devtoolset-10/root/usr/lib - #libxml2 is out of date, compile and install a new one - yum install -y xz - wget https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz - unxz libxml2-2.9.14.tar.xz - tar -xf libxml2-2.9.14.tar - cd libxml2-2.9.14 && ./configure --prefix=/usr/ --without-python && make && make install - cd .. - fi + if [ -d /opt/rh/devtoolset-10/root/usr/lib ]; then + #we are running in the manylinux2014 image + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/opt/rh/devtoolset-10/root/usr/lib + #libxml2 is out of date, compile and install a new one + yum install -y xz + wget https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz + unxz libxml2-2.9.14.tar.xz + tar -xf libxml2-2.9.14.tar + cd libxml2-2.9.14 && ./configure --prefix=/usr/ --without-python && make && make install + cd .. + fi elif [ "$VERSION_ID" = "8" ]; then #they forgot to package libexttextcat-devel? grab one manually: wget https://github.com/proycon/LaMachine/raw/master/deps/centos8/libexttextcat-devel-3.4.5-2.el8.x86_64.rpm From 9da1b26c58fb26d227801b3004986c4a8b4d7f2b Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 23 Jan 2023 14:57:46 +0100 Subject: [PATCH 20/75] version bump and updated ucto dependency --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 3b719f4..d1ee5f8 100755 --- a/setup.py +++ b/setup.py @@ -57,16 +57,16 @@ setup( name = 'python-frog', - version = '0.6.2', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.3', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", - description = ("Python binding to FROG, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), + description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), license = "GPLv3", keywords = "nlp computational_linguistics dutch pos lemmatizer", url = "https://github.com/proycon/python-frog", ext_modules = extensions, cmdclass = {'build_ext': build_ext}, - requires = ['frog (>=0.26)'], + requires = ['frog (>=0.26)','ucto (>=0.27)'], install_requires=['Cython'], data_files = [("sources",["frog_wrapper.pyx"])], classifiers=[ From 06f6af106cea6f9db7586679afe40456bf1fc65f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 23 Jan 2023 15:00:22 +0100 Subject: [PATCH 21/75] removed libtar dependency --- build-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-deps.sh b/build-deps.sh index 0403917..9011d2f 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -50,7 +50,7 @@ fi PWD="$(pwd)" BUILDDIR="$(mktemp -dt "build-deps.XXXXXX")" cd "$BUILDDIR" -for PACKAGE in tklauser/libtar LanguageMachines/ticcutils LanguageMachines/libfolia LanguageMachines/uctodata LanguageMachines/ucto LanguageMachines/timbl LanguageMachines/mbt LanguageMachines/frogdata LanguageMachines/frog; do +for PACKAGE in LanguageMachines/ticcutils LanguageMachines/libfolia LanguageMachines/uctodata LanguageMachines/ucto LanguageMachines/timbl LanguageMachines/mbt LanguageMachines/frogdata LanguageMachines/frog; do echo "Git cloning $PACKAGE ">&2 git clone https://github.com/$PACKAGE PACKAGE="$(basename $PACKAGE)" From cb2321da96f4b93f430e862c0f7443be5a4129d1 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 23 Jan 2023 15:09:49 +0100 Subject: [PATCH 22/75] wheels broke again? try install libxml2 in same prefix as the rest --- build-deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-deps.sh b/build-deps.sh index 9011d2f..04fc8e4 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -21,6 +21,7 @@ get_latest_version() { fi } +[ -z "$PREFIX" ] && PREFIX="/usr/local/" if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then if [ -d /usr/local/share/aclocal ]; then #needed for manylinux_2_28 container which ships custom autoconf, possibly others too? @@ -36,7 +37,7 @@ if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then wget https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz unxz libxml2-2.9.14.tar.xz tar -xf libxml2-2.9.14.tar - cd libxml2-2.9.14 && ./configure --prefix=/usr/ --without-python && make && make install + cd libxml2-2.9.14 && ./configure --prefix=$PREFIX --without-python && make && make install cd .. fi elif [ "$VERSION_ID" = "8" ]; then @@ -46,7 +47,6 @@ if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then fi fi -[ -z "$PREFIX" ] && PREFIX="/usr/local/" PWD="$(pwd)" BUILDDIR="$(mktemp -dt "build-deps.XXXXXX")" cd "$BUILDDIR" From 821b7759d66501a352d01e24ea6ed24396d39da0 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 21 Feb 2023 10:06:17 +0100 Subject: [PATCH 23/75] version bump for compilation against ucto v0.28 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d1ee5f8..45c04eb 100755 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name = 'python-frog', - version = '0.6.3', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.4', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 1b59b76eff51ca19faf38d0b7bd9b7b2216e5ff7 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 22 Feb 2023 12:50:19 +0100 Subject: [PATCH 24/75] version bump for compilation against frog v0.27 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 45c04eb..b943c73 100755 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name = 'python-frog', - version = '0.6.4', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.5', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), @@ -66,7 +66,7 @@ url = "https://github.com/proycon/python-frog", ext_modules = extensions, cmdclass = {'build_ext': build_ext}, - requires = ['frog (>=0.26)','ucto (>=0.27)'], + requires = ['frog (>=0.27)','ucto (>=0.28)'], install_requires=['Cython'], data_files = [("sources",["frog_wrapper.pyx"])], classifiers=[ From 00f17117a7efc59825ada88fe35b7be50d7f714b Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 22 Feb 2023 12:51:41 +0100 Subject: [PATCH 25/75] added codemeta-harvest.json --- codemeta-harvest.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 codemeta-harvest.json diff --git a/codemeta-harvest.json b/codemeta-harvest.json new file mode 100644 index 0000000..bc4bc63 --- /dev/null +++ b/codemeta-harvest.json @@ -0,0 +1,18 @@ +{ + "funding": { + "@type": "Grant", + "name": "CLARIAH-PLUS (NWO grant 184.034.023)", + "funder": { + "@type": "Organization", + "name": "NWO", + "url": "https://www.nwo.nl" + } + }, + "producer": { + "@type": "Organization", + "name": "KNAW Humanities Cluster", + "url": "https://huc.knaw.nl" + }, + "applicationCategory": [ "https://vocabs.dariah.eu/tadirah/annotating", "https://vocabs.dariah.eu/tadirah/tagging", "https://vocabs.dariah.eu/tadirah/namedEntityRecognition", "https://vocabs.dariah.eu/tadirah/posTagging", "https://vocabs.dariah.eu/tadirah/segmenting", "https://vocabs.dariah.eu/tadirah/treeTagging", "https://vocabs.dariah.eu/tadirah/contextualizing" , "https://w3id.org/nwo-research-fields#Linguistics", "https://w3id.org/nwo-research-fields#TextualAndContentAnalysis" ], + "developmentStatus": [ "https://www.repostatus.org/#active", "https://w3id.org/research-technology-readiness-levels#Level9Proven" ], +} From e6770e77683909d5b26ec82e6ced0b30f012423c Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 22 Feb 2023 12:54:26 +0100 Subject: [PATCH 26/75] metadata update --- codemeta-harvest.json | 1 + 1 file changed, 1 insertion(+) diff --git a/codemeta-harvest.json b/codemeta-harvest.json index bc4bc63..a770f0e 100644 --- a/codemeta-harvest.json +++ b/codemeta-harvest.json @@ -13,6 +13,7 @@ "name": "KNAW Humanities Cluster", "url": "https://huc.knaw.nl" }, + "contIntegration": "https://github.com/proycon/python-frog/actions/workflows/python-frog.yml", "applicationCategory": [ "https://vocabs.dariah.eu/tadirah/annotating", "https://vocabs.dariah.eu/tadirah/tagging", "https://vocabs.dariah.eu/tadirah/namedEntityRecognition", "https://vocabs.dariah.eu/tadirah/posTagging", "https://vocabs.dariah.eu/tadirah/segmenting", "https://vocabs.dariah.eu/tadirah/treeTagging", "https://vocabs.dariah.eu/tadirah/contextualizing" , "https://w3id.org/nwo-research-fields#Linguistics", "https://w3id.org/nwo-research-fields#TextualAndContentAnalysis" ], "developmentStatus": [ "https://www.repostatus.org/#active", "https://w3id.org/research-technology-readiness-levels#Level9Proven" ], } From 6005296f2bd99408d755c8692e12d0ad728a25f0 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 23 Feb 2023 14:12:23 +0100 Subject: [PATCH 27/75] Allowing having textcat data in a non-standard location #23 --- frog_wrapper.pyx | 10 ++++++++-- setup.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index c5340ad..d12947c 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -255,11 +255,17 @@ def installdata(targetdir=None, frogdataversion=FROGDATAVERSION, uctodataversion raise Exception("Installation of frogdata failed") print(f"Installation of frogdata {frogdataversion} complete", file=sys.stderr) - if os.path.isdir("/usr/share/libexttextcat"): + textcatpath = os.environ.get("TEXTCATPATH","/usr/share/libexttextcat") + if os.path.isdir(textcatpath): if os.system(f"cd {uctodir} && wget -O textcat.cfg https://raw.githubusercontent.com/LanguageMachines/ucto/master/config/textcat.cfg") != 0: raise Exception("Installation of textcat.cfg failed.") + if textcatpath[-1] = '/': textcatpath = textcatpath[:-1] + if textcatpath != "/usr/share/libexttextcat": + data = open(f"{uctodir}/textcat.cfg",'r',encoding='utf-8').read() + with open(f"{uctodir}/textcat.cfg",'w',encoding='utf-8').read() + f.write(data.replace("/usr/share/libexttextcat", textcatpath)) else: - print("Language detection will not be available unless you install libexttextcat and rerun installdata()", file=sys.stderr) + print("Language detection will not be available unless you install libexttextcat and rerun installdata(). Libexttextcat was not found. If it is in a non-standard location, set environment variable TEXTCATPATH (defaults to /usr/share/libexttextcat) prior to calling installdata()", file=sys.stderr) diff --git a/setup.py b/setup.py index b943c73..4dcec20 100755 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name = 'python-frog', - version = '0.6.5', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.6', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From b86cf6b41349816ae2998f5b939873e70280e86e Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 23 Feb 2023 14:42:10 +0100 Subject: [PATCH 28/75] fixup: syntax --- frog_wrapper.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index d12947c..8509532 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -259,7 +259,7 @@ def installdata(targetdir=None, frogdataversion=FROGDATAVERSION, uctodataversion if os.path.isdir(textcatpath): if os.system(f"cd {uctodir} && wget -O textcat.cfg https://raw.githubusercontent.com/LanguageMachines/ucto/master/config/textcat.cfg") != 0: raise Exception("Installation of textcat.cfg failed.") - if textcatpath[-1] = '/': textcatpath = textcatpath[:-1] + if textcatpath[-1] == '/': textcatpath = textcatpath[:-1] if textcatpath != "/usr/share/libexttextcat": data = open(f"{uctodir}/textcat.cfg",'r',encoding='utf-8').read() with open(f"{uctodir}/textcat.cfg",'w',encoding='utf-8').read() From 8f02a8c8e36124278c55d5613b0e6d7c60bd8655 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 23 Feb 2023 18:09:41 +0100 Subject: [PATCH 29/75] fixup --- frog_wrapper.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index 8509532..7f55bbc 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -262,7 +262,7 @@ def installdata(targetdir=None, frogdataversion=FROGDATAVERSION, uctodataversion if textcatpath[-1] == '/': textcatpath = textcatpath[:-1] if textcatpath != "/usr/share/libexttextcat": data = open(f"{uctodir}/textcat.cfg",'r',encoding='utf-8').read() - with open(f"{uctodir}/textcat.cfg",'w',encoding='utf-8').read() + with open(f"{uctodir}/textcat.cfg",'w',encoding='utf-8'): f.write(data.replace("/usr/share/libexttextcat", textcatpath)) else: print("Language detection will not be available unless you install libexttextcat and rerun installdata(). Libexttextcat was not found. If it is in a non-standard location, set environment variable TEXTCATPATH (defaults to /usr/share/libexttextcat) prior to calling installdata()", file=sys.stderr) From c94bd8155520a7c2ef768397c68a912bad67483f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 23 Feb 2023 18:28:44 +0100 Subject: [PATCH 30/75] fixup again, I keep messing up --- frog_wrapper.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index 7f55bbc..167d2f4 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -262,7 +262,7 @@ def installdata(targetdir=None, frogdataversion=FROGDATAVERSION, uctodataversion if textcatpath[-1] == '/': textcatpath = textcatpath[:-1] if textcatpath != "/usr/share/libexttextcat": data = open(f"{uctodir}/textcat.cfg",'r',encoding='utf-8').read() - with open(f"{uctodir}/textcat.cfg",'w',encoding='utf-8'): + with open(f"{uctodir}/textcat.cfg",'w',encoding='utf-8') as f: f.write(data.replace("/usr/share/libexttextcat", textcatpath)) else: print("Language detection will not be available unless you install libexttextcat and rerun installdata(). Libexttextcat was not found. If it is in a non-standard location, set environment variable TEXTCATPATH (defaults to /usr/share/libexttextcat) prior to calling installdata()", file=sys.stderr) From 8e9b464b92848675783448c7621f76d9825e65bd Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 24 May 2023 14:24:26 +0200 Subject: [PATCH 31/75] version bump for compilation against frog v0.30 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4dcec20..a04e330 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ url = "https://github.com/proycon/python-frog", ext_modules = extensions, cmdclass = {'build_ext': build_ext}, - requires = ['frog (>=0.27)','ucto (>=0.28)'], + requires = ['frog (>=0.30)','ucto (>=0.29)'], install_requires=['Cython'], data_files = [("sources",["frog_wrapper.pyx"])], classifiers=[ From 91749c3eabf80c7197db8c565e675508118bf6f1 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 29 Jun 2023 16:02:53 +0200 Subject: [PATCH 32/75] ci: wheel building, why no cython on mac? --- .github/workflows/wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e96b262..222b972 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -38,7 +38,8 @@ jobs: ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > brew tap fbkarsdorp/homebrew-lamachine && - brew install frog + brew install frog && + pip install cython CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" From 858dbc2cb7a4465b183805c3a0f9ed83725e7c32 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 29 Jun 2023 16:08:30 +0200 Subject: [PATCH 33/75] Revert "ci: wheel building, why no cython on mac?" This reverts commit 91749c3eabf80c7197db8c565e675508118bf6f1. --- .github/workflows/wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 222b972..e96b262 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -38,8 +38,7 @@ jobs: ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > brew tap fbkarsdorp/homebrew-lamachine && - brew install frog && - pip install cython + brew install frog CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" From 2259f8384aea5cd2edbe7c95cadc20ad1cfe7122 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 29 Jun 2023 16:29:02 +0200 Subject: [PATCH 34/75] added gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..40da0e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +build/ +env/ +*.egg-info/ +*.py[cod] +*~ From 9045b362021bf9cc9d03de06aad83eae521a4517 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 3 Jul 2023 22:12:10 +0200 Subject: [PATCH 35/75] README: added demo screencast --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d22a8c1..bf37abf 100644 --- a/README.rst +++ b/README.rst @@ -161,5 +161,7 @@ An example can be found below: for word in output.words(): print(word.text() + " " + word.pos() + " " + word.lemma()) +Demo +------------------ - +.. image:: https://raw.githubusercontent.com/CLARIAH/wp3-demos/master/python-frog.gif From 7bd4f4cadc9cad3d703f292f818294ccfbc36526 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 3 Jul 2023 22:15:14 +0200 Subject: [PATCH 36/75] README: moved demo up --- README.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index bf37abf..f567510 100644 --- a/README.rst +++ b/README.rst @@ -17,6 +17,11 @@ morphological analysis, named entity recognition, shallow parsing, and dependency parsing. The tool itself is implemented in C++ (https://languagemachines.github.io/frog). The binding requires Python 3.6 or higher. +Demo +------------------ + +.. image:: https://raw.githubusercontent.com/CLARIAH/wp3-demos/master/python-frog.gif + Installation ---------------- @@ -161,7 +166,3 @@ An example can be found below: for word in output.words(): print(word.text() + " " + word.pos() + " " + word.lemma()) -Demo ------------------- - -.. image:: https://raw.githubusercontent.com/CLARIAH/wp3-demos/master/python-frog.gif From 0b3e0513c7a290112e0449282f6c4be64eb0661b Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 11 Sep 2023 10:55:11 +0200 Subject: [PATCH 37/75] version bump for new wheel building, no functional changes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a04e330..f1cd9cf 100755 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name = 'python-frog', - version = '0.6.6', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.7', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From d24b02be4159527f4fd29fba94061696b7be16a3 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 11 Sep 2023 11:41:15 +0200 Subject: [PATCH 38/75] cythonize() takes only pyx and breaks on pxd now? #25 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f1cd9cf..8b0b8f8 100755 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ extensions = cythonize([ Extension("frog", - [ "libfolia_classes.pxd", "frog_classes.pxd", "frog_wrapper.pyx"], + [ "frog_wrapper.pyx"], language='c++', include_dirs=includedirs, library_dirs=libdirs, From c7ce86647b8adecceb8374ab8c642da4137d337e Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 11 Sep 2023 13:27:59 +0200 Subject: [PATCH 39/75] wheel building: try newer macos ci runner --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e96b262..8d4972f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macOS-11] + os: [ubuntu-20.04, macOS-12] steps: - uses: actions/checkout@v3 From bc7cbb5d8d5b2d0bbb21bd3a97365156ba359389 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 11:30:42 +0200 Subject: [PATCH 40/75] wheel building: build frog from source instead of brew (as recommended by cibuildwheel) --- .github/workflows/wheels.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8d4972f..036e169 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -37,8 +37,15 @@ jobs: fi && ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > - brew tap fbkarsdorp/homebrew-lamachine && - brew install frog + brew install pkg-config icu4c && + git clone https://gitlab.gnome.org/GNOME/libxml2 && + cd libxml2 && + autoreconf --install && ./configure && make && make install && + git clone https://github.com/LanguageMachines/frog && + cd frog && + ./build-deps.sh && + ./configure && + make && make install CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" From 21ef37d8de330fc18c60f9a4bf653f192d32d181 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 11:34:54 +0200 Subject: [PATCH 41/75] macos wheel building: added autoconf-archive --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 036e169..ac1dbc6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -37,7 +37,7 @@ jobs: fi && ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > - brew install pkg-config icu4c && + brew install pkg-config icu4c autoconf-archive && git clone https://gitlab.gnome.org/GNOME/libxml2 && cd libxml2 && autoreconf --install && ./configure && make && make install && From 895f5b79e78eec56a5bfdf91d654677c8348565d Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 11:39:42 +0200 Subject: [PATCH 42/75] macos wheel building: more deps --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ac1dbc6..392a138 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -37,7 +37,7 @@ jobs: fi && ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > - brew install pkg-config icu4c autoconf-archive && + brew install pkg-config icu4c autoconf-archive autoconf automake && git clone https://gitlab.gnome.org/GNOME/libxml2 && cd libxml2 && autoreconf --install && ./configure && make && make install && From 824b1ad5f61b27d15646e9722464488aa46f3d26 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 12:19:14 +0200 Subject: [PATCH 43/75] macos wheel building: compile icu from source --- .github/workflows/wheels.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 392a138..94bda0b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -37,15 +37,19 @@ jobs: fi && ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > - brew install pkg-config icu4c autoconf-archive autoconf automake && + brew install pkg-config autoconf-archive autoconf automake && + git clone https://github.com/unicode-org/icu && + cd icu/icu4c/source && + ./configure && make && make install && + cd ../../../ && git clone https://gitlab.gnome.org/GNOME/libxml2 && cd libxml2 && autoreconf --install && ./configure && make && make install && git clone https://github.com/LanguageMachines/frog && cd frog && ./build-deps.sh && - ./configure && - make && make install + ./bootstrap.sh && + ./configure && make && make install CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" From 07a80f677503d60be425b4c993569a0d758f5a67 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 12:55:46 +0200 Subject: [PATCH 44/75] macos wheel building: compile libtextcat from source --- .github/workflows/wheels.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 94bda0b..73ab76e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -45,6 +45,11 @@ jobs: git clone https://gitlab.gnome.org/GNOME/libxml2 && cd libxml2 && autoreconf --install && ./configure && make && make install && + cd ../ && + wget https://software.wise-guys.nl/download/libtextcat-2.2.tar.gz && + tar xzf libtextcat-2.2.tar.gz && + cd libtextcat && autoreconf -i && ./configure && make && make install && + cd ../ && git clone https://github.com/LanguageMachines/frog && cd frog && ./build-deps.sh && From 4e805709769328affb84825cf8842a3a3d623aa9 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 12:56:24 +0200 Subject: [PATCH 45/75] fixup --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 73ab76e..3106637 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -48,7 +48,7 @@ jobs: cd ../ && wget https://software.wise-guys.nl/download/libtextcat-2.2.tar.gz && tar xzf libtextcat-2.2.tar.gz && - cd libtextcat && autoreconf -i && ./configure && make && make install && + cd libtextcat-2.2 && autoreconf -i && ./configure && make && make install && cd ../ && git clone https://github.com/LanguageMachines/frog && cd frog && From 55a84d61e254a763f687d606e19458cf1f7b6c3f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 13:45:12 +0200 Subject: [PATCH 46/75] macos wheel building: libtextcat compilation --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3106637..92a3f64 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -49,6 +49,7 @@ jobs: wget https://software.wise-guys.nl/download/libtextcat-2.2.tar.gz && tar xzf libtextcat-2.2.tar.gz && cd libtextcat-2.2 && autoreconf -i && ./configure && make && make install && + mkdir /usr/local/include/libtextcat/ && cp src/*.h /usr/local/include/libtextcat/ && cd ../ && git clone https://github.com/LanguageMachines/frog && cd frog && From 788143cb2172de73931ef51dce289b7811b77507 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 15:02:07 +0200 Subject: [PATCH 47/75] macos wheel building: compile xz (liblzma.dylib) from source --- .github/workflows/wheels.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 92a3f64..2a989b3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -38,9 +38,13 @@ jobs: ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > brew install pkg-config autoconf-archive autoconf automake && + git clone https://github.com/tukaani-project/xz && + cd xz && + autoreconf --install && ./configure && make && make install && + cd .. git clone https://github.com/unicode-org/icu && cd icu/icu4c/source && - ./configure && make && make install && + ./configure && make && make install && ls -l /usr/local/lib/libicu* && cd ../../../ && git clone https://gitlab.gnome.org/GNOME/libxml2 && cd libxml2 && From 7d19591eac1ead81ee584fe1a23151b44d1982e2 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 12 Sep 2023 15:02:38 +0200 Subject: [PATCH 48/75] fixup --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2a989b3..9954741 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -41,7 +41,7 @@ jobs: git clone https://github.com/tukaani-project/xz && cd xz && autoreconf --install && ./configure && make && make install && - cd .. + cd .. && git clone https://github.com/unicode-org/icu && cd icu/icu4c/source && ./configure && make && make install && ls -l /usr/local/lib/libicu* && From 6dfdf9c754d0b5a974453577436c588f01471d2a Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Sep 2023 10:12:09 +0200 Subject: [PATCH 49/75] reverting all of yesterday's changes --- .github/workflows/wheels.yml | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9954741..e96b262 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macOS-12] + os: [ubuntu-20.04, macOS-11] steps: - uses: actions/checkout@v3 @@ -37,29 +37,8 @@ jobs: fi && ./build-deps.sh CIBW_BEFORE_ALL_MACOS: > - brew install pkg-config autoconf-archive autoconf automake && - git clone https://github.com/tukaani-project/xz && - cd xz && - autoreconf --install && ./configure && make && make install && - cd .. && - git clone https://github.com/unicode-org/icu && - cd icu/icu4c/source && - ./configure && make && make install && ls -l /usr/local/lib/libicu* && - cd ../../../ && - git clone https://gitlab.gnome.org/GNOME/libxml2 && - cd libxml2 && - autoreconf --install && ./configure && make && make install && - cd ../ && - wget https://software.wise-guys.nl/download/libtextcat-2.2.tar.gz && - tar xzf libtextcat-2.2.tar.gz && - cd libtextcat-2.2 && autoreconf -i && ./configure && make && make install && - mkdir /usr/local/include/libtextcat/ && cp src/*.h /usr/local/include/libtextcat/ && - cd ../ && - git clone https://github.com/LanguageMachines/frog && - cd frog && - ./build-deps.sh && - ./bootstrap.sh && - ./configure && make && make install + brew tap fbkarsdorp/homebrew-lamachine && + brew install frog CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" From 3e61e747b8d5248e70f7f4b5a5b78244429d856f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Sep 2023 10:24:12 +0200 Subject: [PATCH 50/75] macos wheel building: switch to cirrus ci for native macos arm64 runners --- .cirrus.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .cirrus.yml diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000..c09d039 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,24 @@ +######################################################### +# Build arm64 wheels for OSX on Cirrus CI +######################################################### + +cirrus_wheels_macos_arm64_task: + name: Build macOS arm64 wheels. + trigger_type: manual + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:13.3.1 + env: + PATH: /opt/homebrew/opt/python@3.10/bin:$PATH + CIBW_ARCHS_MACOS: arm64 + install_pre_requirements_script: + - brew install python@3.10 + - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python + - which python + - python --version + install_cibuildwheel_script: + - python -m pip install cibuildwheel + run_cibuildwheel_script: + - CIBW_SKIP="*-win32 *-manylinux_* *-musllinux_*" CIBW_ARCHS_MACOS="arm64" CIBW_BUILD="cp39-* cp310-* cp311-*" CIBW_BEFORE_ALL_MACOS="brew tap fbkarsdorp/homebrew-lamachine && brew install frog" cibuildwheel --platform macos + wheels_artifacts: + path: "wheelhouse/*" + From fbad767e937607d293a12c9c0e7a910093f3c2f7 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Sep 2023 10:53:02 +0200 Subject: [PATCH 51/75] wheel building debug: increase verbosity --- .cirrus.yml | 3 +-- .github/workflows/wheels.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index c09d039..3db002a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,7 +4,6 @@ cirrus_wheels_macos_arm64_task: name: Build macOS arm64 wheels. - trigger_type: manual macos_instance: image: ghcr.io/cirruslabs/macos-monterey-xcode:13.3.1 env: @@ -18,7 +17,7 @@ cirrus_wheels_macos_arm64_task: install_cibuildwheel_script: - python -m pip install cibuildwheel run_cibuildwheel_script: - - CIBW_SKIP="*-win32 *-manylinux_* *-musllinux_*" CIBW_ARCHS_MACOS="arm64" CIBW_BUILD="cp39-* cp310-* cp311-*" CIBW_BEFORE_ALL_MACOS="brew tap fbkarsdorp/homebrew-lamachine && brew install frog" cibuildwheel --platform macos + - CIBW_BUILD_VERBOSITY=3 CIBW_SKIP="*-win32 *-manylinux_* *-musllinux_*" CIBW_ARCHS_MACOS="arm64" CIBW_BUILD="cp39-* cp310-* cp311-*" CIBW_BEFORE_ALL_MACOS="brew tap fbkarsdorp/homebrew-lamachine && brew install frog" cibuildwheel --platform macos wheels_artifacts: path: "wheelhouse/*" diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e96b262..db3a0ac 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -25,7 +25,7 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_ARCHS_LINUX: "x86_64" - CIBW_ARCHS_MACOS: "x86_64 arm64" + CIBW_ARCHS_MACOS: "x86_64" CIBW_BEFORE_ALL_LINUX: > if command -v apt-get; then apt-get -y git libicu-dev libxml2-dev libexttextcat-dev libxslt1-dev libbz2-dev zlib1g-dev autoconf automake autoconf-archive libtool autotools-dev gcc g++ make From ed7ab8d22267b12ec66bafa9fd4e4223275adba6 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Sep 2023 11:02:31 +0200 Subject: [PATCH 52/75] debug setup.py --- .cirrus.yml | 3 ++- setup.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 3db002a..4c8998f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,8 +4,9 @@ cirrus_wheels_macos_arm64_task: name: Build macOS arm64 wheels. + trigger_type: manual macos_instance: - image: ghcr.io/cirruslabs/macos-monterey-xcode:13.3.1 + image: ghcr.io/cirruslabs/macos-monterey-xcode:latest env: PATH: /opt/homebrew/opt/python@3.10/bin:$PATH CIBW_ARCHS_MACOS: arm64 diff --git a/setup.py b/setup.py index 8b0b8f8..90face3 100755 --- a/setup.py +++ b/setup.py @@ -4,13 +4,16 @@ from Cython.Build import cythonize import platform import os +import sys includedirs = [] libdirs = [] +print(f"system={platform.system()} machine={platform.machine()}", file=sys.stderr) if platform.system() == "Darwin": #we are running on Mac OS X (with homebrew hopefully), stuff is in specific locations: if platform.machine().lower() == "arm64": + print("(macos arm64 detected)", file=sys.stderr) libdirs.append("/opt/homebrew/lib") includedirs.append("/opt/homebrew/include") libdirs.append("/opt/homebrew/icu4c/lib") @@ -42,6 +45,8 @@ else: extra_options = ['-D U_USING_ICU_NAMESPACE=1'] +print(f"include_dirs={' '.join(includedirs)} library_dirs={' '.join(libdirs)} extra_options={' '.join(extra_options)}", file=sys.stderr) + extensions = cythonize([ Extension("frog", [ "frog_wrapper.pyx"], From 3d25acc2154f9dd97ccbd234e439ca890813e851 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Sep 2023 11:09:36 +0200 Subject: [PATCH 53/75] macos wheel building: explicitly install icu4c and libxml2 --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 4c8998f..3468577 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -18,7 +18,7 @@ cirrus_wheels_macos_arm64_task: install_cibuildwheel_script: - python -m pip install cibuildwheel run_cibuildwheel_script: - - CIBW_BUILD_VERBOSITY=3 CIBW_SKIP="*-win32 *-manylinux_* *-musllinux_*" CIBW_ARCHS_MACOS="arm64" CIBW_BUILD="cp39-* cp310-* cp311-*" CIBW_BEFORE_ALL_MACOS="brew tap fbkarsdorp/homebrew-lamachine && brew install frog" cibuildwheel --platform macos + - CIBW_BUILD_VERBOSITY=3 CIBW_SKIP="*-win32 *-manylinux_* *-musllinux_*" CIBW_ARCHS_MACOS="arm64" CIBW_BUILD="cp39-* cp310-* cp311-*" CIBW_BEFORE_ALL_MACOS="brew tap fbkarsdorp/homebrew-lamachine && brew install icu4c libxml2 frog" cibuildwheel --platform macos wheels_artifacts: path: "wheelhouse/*" From 023b89cb1fe3e9aabd26765f5ac4231e779d52d0 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Wed, 13 Sep 2023 11:14:35 +0200 Subject: [PATCH 54/75] macosx wheel building: compile against homebrew's 'keg-only' libxml2 and icu4c --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 90face3..be70787 100755 --- a/setup.py +++ b/setup.py @@ -21,6 +21,10 @@ libdirs.append("/opt/homebrew/libxml2/lib") includedirs.append("/opt/homebrew/libxml2/include") includedirs.append("/opt/homebrew/libxml2/include/libxml2") + libdirs.append("/opt/homebrew/opt/icu4c/lib") + includedirs.append("/opt/homebrew/opt/icu4c/include") + libdirs.append("/opt/homebrew/opt/libxml2/lib") + includedirs.append("/opt/homebrew/opt/libxml2/include") else: #we are running on Mac OS X with homebrew, stuff is in specific locations: libdirs.append("/usr/local/opt/icu4c/lib") From 1babaa474b127bc2f6fa5ec4b0ed66be4eb10d89 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 31 Oct 2023 16:11:47 +0100 Subject: [PATCH 55/75] updated dependencies for frog 0.31 --- frog_wrapper.pyx | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index 167d2f4..84ca1ba 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -20,7 +20,7 @@ import sys cimport libfolia_classes cimport frog_classes -FROGDATAVERSION = "0.21" +FROGDATAVERSION = "0.22" UCTODATAVERSION = "0.9.1" try: diff --git a/setup.py b/setup.py index be70787..faa4735 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name = 'python-frog', - version = '0.6.7', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.8', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From e169d84bf52a488ccee9aded27100bae0ab471c4 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 20 Nov 2023 12:10:10 +0100 Subject: [PATCH 56/75] Allow passing docid as string instead of bytes #26 For backward-compatibility, passing bytes is still allowed as well --- frog_wrapper.pyx | 5 ++++- setup.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index 84ca1ba..fc5ac48 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -122,7 +122,10 @@ cdef class FrogOptions: elif key.lower() in ('debug','debugflag'): if value: self.capi.insert(b"d", b"1", False) elif key.lower() in ('docid','id'): - self.capi.insert(b"id", value) + if isinstance(value, bytes): + self.capi.insert(b"id", value) + else: + self.capi.insert(b"id", value.encode('utf-8')) elif key.lower() in ('numthreads','threads'): self.capi.insert(b"threads",value) else: diff --git a/setup.py b/setup.py index faa4735..e8c265e 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name = 'python-frog', - version = '0.6.8', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.9', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 5d57c18ca7302e94a2ee21ca2005bab58705978f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 27 Nov 2023 15:03:38 +0100 Subject: [PATCH 57/75] codemeta fix --- codemeta-harvest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codemeta-harvest.json b/codemeta-harvest.json index a770f0e..767e19e 100644 --- a/codemeta-harvest.json +++ b/codemeta-harvest.json @@ -15,5 +15,5 @@ }, "contIntegration": "https://github.com/proycon/python-frog/actions/workflows/python-frog.yml", "applicationCategory": [ "https://vocabs.dariah.eu/tadirah/annotating", "https://vocabs.dariah.eu/tadirah/tagging", "https://vocabs.dariah.eu/tadirah/namedEntityRecognition", "https://vocabs.dariah.eu/tadirah/posTagging", "https://vocabs.dariah.eu/tadirah/segmenting", "https://vocabs.dariah.eu/tadirah/treeTagging", "https://vocabs.dariah.eu/tadirah/contextualizing" , "https://w3id.org/nwo-research-fields#Linguistics", "https://w3id.org/nwo-research-fields#TextualAndContentAnalysis" ], - "developmentStatus": [ "https://www.repostatus.org/#active", "https://w3id.org/research-technology-readiness-levels#Level9Proven" ], + "developmentStatus": [ "https://www.repostatus.org/#active", "https://w3id.org/research-technology-readiness-levels#Level9Proven" ] } From 976801722deae5d2a5285f1f0bcfc476f7ae4436 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 5 Dec 2023 15:47:42 +0100 Subject: [PATCH 58/75] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8c265e..81c086c 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name = 'python-frog', - version = '0.6.9', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.10', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 33991c0085f67808312c5999ec2ec81186b5adc5 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 12 Sep 2024 15:46:48 +0200 Subject: [PATCH 59/75] ci: updaded wheel building --- .github/workflows/wheels.yml | 42 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index db3a0ac..d9e754a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -6,42 +6,64 @@ on: [workflow_dispatch] jobs: build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build wheels on for ${{matrix.python.cp}}-${{ matrix.buildplat.sys }} + runs-on: ${{ matrix.buildplat.runs_on }} strategy: matrix: - os: [ubuntu-20.04, macOS-11] + buildplat: + - { runs_on: ubuntu-20.04, sys: manylinux, arch: x86_64, benv: "" } + - { runs_on: macos-12, sys: macosx, arch: x86_64, benv: "12.0" } + - { runs_on: macos-14, sys: macosx, arch: arm64, benv: "14.0" } + python: + - { cp: "cp38", rel: "3.8" } + - { cp: "cp39", rel: "3.9" } + - { cp: "cp310", rel: "3.10" } + - { cp: "cp311", rel: "3.11" } + - { cp: "cp312", rel: "3.12" } steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.1 # Used to host cibuildwheel - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 + with: + python-version: 3.11 - name: Install cibuildwheel run: python -m pip install cibuildwheel - - name: Build wheels + - name: Build wheels (Linux) + if: ${{ runner.os != 'macOS' }} run: python -m cibuildwheel --output-dir wheelhouse env: + CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* CIBW_ARCHS_LINUX: "x86_64" - CIBW_ARCHS_MACOS: "x86_64" CIBW_BEFORE_ALL_LINUX: > if command -v apt-get; then apt-get -y git libicu-dev libxml2-dev libexttextcat-dev libxslt1-dev libbz2-dev zlib1g-dev autoconf automake autoconf-archive libtool autotools-dev gcc g++ make elif command -v yum; then yum install -y git libicu-devel libxml2-devel libxslt-devel libexttextcat zlib-devel bzip2-devel libtool autoconf-archive autoconf automake m4 wget + #on CentOS 7 we also have libtar-devel libexttextcat-devel which will be installed by build-deps.sh, on 8 they are missing and will be installed from source or otherwise elif command -v apk; then apk add build-base git autoconf-archive autoconf automake libtool bzip2-dev icu-dev libxml2-dev libexttextcat-dev libtool rsync && rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ fi && ./build-deps.sh + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 + CIBW_SKIP: "*-win* *-manylinux_i686 pp*" + + - name: Build wheels (macOS) + if: ${{ runner.os == 'macOS' }} + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* + CIBW_ARCHS: ${{ matrix.buildplat.arch }} + CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat.benv }}" CIBW_BEFORE_ALL_MACOS: > brew tap fbkarsdorp/homebrew-lamachine && brew install frog - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 - CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: + name: ${{matrix.python.cp}}-${{matrix.buildplat.sys}}-${{matrix.buildplat.arch}} path: ./wheelhouse/*.whl From 6b56ff6b7c688b7d42aedf27d91c7672528776c4 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 12 Sep 2024 16:01:55 +0200 Subject: [PATCH 60/75] ci: skip python3.8 on mac --- .github/workflows/wheels.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index d9e754a..c6c934a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -53,7 +53,7 @@ jobs: CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - name: Build wheels (macOS) - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os == 'macOS' && runner.python != "3.8" }} run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* @@ -64,6 +64,7 @@ jobs: brew install frog - uses: actions/upload-artifact@v4 + if: ${{ ! (runner.os == 'macOS' && runner.python == "3.8") }} with: name: ${{matrix.python.cp}}-${{matrix.buildplat.sys}}-${{matrix.buildplat.arch}} path: ./wheelhouse/*.whl From 02946db86196b25270972fe316bb6414290d079c Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 28 Oct 2024 17:28:59 +0100 Subject: [PATCH 61/75] ci: syntax fix something must have changed github-side --- .github/workflows/wheels.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index c6c934a..b08f2a6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -20,6 +20,7 @@ jobs: - { cp: "cp310", rel: "3.10" } - { cp: "cp311", rel: "3.11" } - { cp: "cp312", rel: "3.12" } + - { cp: "cp313", rel: "3.13" } steps: - uses: actions/checkout@v4.1.1 @@ -53,7 +54,7 @@ jobs: CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - name: Build wheels (macOS) - if: ${{ runner.os == 'macOS' && runner.python != "3.8" }} + if: ${{ runner.os == 'macOS' && runner.python != '3.8' }} run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* @@ -64,7 +65,7 @@ jobs: brew install frog - uses: actions/upload-artifact@v4 - if: ${{ ! (runner.os == 'macOS' && runner.python == "3.8") }} + if: ${{ ! (runner.os == 'macOS' && runner.python == '3.8') }} with: name: ${{matrix.python.cp}}-${{matrix.buildplat.sys}}-${{matrix.buildplat.arch}} path: ./wheelhouse/*.whl From 873ff9b0594f3102b285b0aae3093f2f7e70d4b4 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 17 Dec 2024 12:37:54 +0100 Subject: [PATCH 62/75] ci: wheel building update --- .github/workflows/wheels.yml | 27 +++++++++++++++++------ build-deps.sh | 42 ++++++++++++++++++++---------------- setup.py | 2 +- 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b08f2a6..992f55e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -12,7 +12,8 @@ jobs: matrix: buildplat: - { runs_on: ubuntu-20.04, sys: manylinux, arch: x86_64, benv: "" } - - { runs_on: macos-12, sys: macosx, arch: x86_64, benv: "12.0" } + - { runs_on: ubuntu-20.04, sys: musllinux, arch: x86_64, benv: "" } + - { runs_on: ubuntu-20.04, sys: musllinux, arch: aarch64, benv: "" } - { runs_on: macos-14, sys: macosx, arch: arm64, benv: "14.0" } python: - { cp: "cp38", rel: "3.8" } @@ -33,8 +34,8 @@ jobs: - name: Install cibuildwheel run: python -m pip install cibuildwheel - - name: Build wheels (Linux) - if: ${{ runner.os != 'macOS' }} + - name: Build wheels (Linux glibc) + if: ${{ matrix.buildplat.sys == 'manylinux' }} run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* @@ -50,11 +51,25 @@ jobs: rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ fi && ./build-deps.sh - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux2014_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 + CIBW_SKIP: "*-win* *-manylinux_i686 pp*" + + - name: Build wheels (Linux musl) + if: ${{ matrix.buildplat.sys == 'musllinux' }} + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* + CIBW_ARCHS_LINUX: "x86_64 aarch64" + CIBW_BEFORE_ALL_LINUX: > + apk add build-base git autoconf-archive autoconf automake libtool bzip2-dev icu-dev libxml2-dev libexttextcat-dev libtool rsync && + mkdir -p /usr/local/share/aclocal/ && rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ && + ./build-deps.sh + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/musllinux_1_1_x86_64 + CIBW_MUSLLINUX_AARCH64_IMAGE: quay.io/pypa/musllinux_1_1_aarch64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - name: Build wheels (macOS) - if: ${{ runner.os == 'macOS' && runner.python != '3.8' }} + if: ${{ runner.os == 'macOS' && matrix.python.cp != 'cp38' }} run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* @@ -65,7 +80,7 @@ jobs: brew install frog - uses: actions/upload-artifact@v4 - if: ${{ ! (runner.os == 'macOS' && runner.python == '3.8') }} + if: ${{ ! (runner.os == 'macOS' && matrix.python.cp == '3.8') }} with: name: ${{matrix.python.cp}}-${{matrix.buildplat.sys}}-${{matrix.buildplat.arch}} path: ./wheelhouse/*.whl diff --git a/build-deps.sh b/build-deps.sh index 04fc8e4..0173d14 100755 --- a/build-deps.sh +++ b/build-deps.sh @@ -7,6 +7,8 @@ set -e . /etc/os-release +echo "OS: $ID">&2 +echo "VERSION: $VERSION_ID">&2 get_latest_version() { #Finds the latest git tag or falls back to returning the git default branch (usually master or main) @@ -27,24 +29,27 @@ if [ "$ID" = "almalinux" ] || [ "$ID" = "centos" ] || [ "$ID" = "rhel" ]; then #needed for manylinux_2_28 container which ships custom autoconf, possibly others too? export ACLOCAL_PATH=/usr/share/aclocal fi - if [ "$VERSION_ID" = "7" ]; then - yum install -y libexttextcat-devel - if [ -d /opt/rh/devtoolset-10/root/usr/lib ]; then - #we are running in the manylinux2014 image - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/opt/rh/devtoolset-10/root/usr/lib - #libxml2 is out of date, compile and install a new one - yum install -y xz - wget https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz - unxz libxml2-2.9.14.tar.xz - tar -xf libxml2-2.9.14.tar - cd libxml2-2.9.14 && ./configure --prefix=$PREFIX --without-python && make && make install - cd .. - fi - elif [ "$VERSION_ID" = "8" ]; then - #they forgot to package libexttextcat-devel? grab one manually: - wget https://github.com/proycon/LaMachine/raw/master/deps/centos8/libexttextcat-devel-3.4.5-2.el8.x86_64.rpm - yum install -y libexttextcat-devel-3.4.5-2.el8.x86_64.rpm - fi + case $VERSION_ID in + 7*) + yum install -y libexttextcat-devel + if [ -d /opt/rh/devtoolset-10/root/usr/lib ]; then + #we are running in the manylinux2014 image + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/opt/rh/devtoolset-10/root/usr/lib + #libxml2 is out of date, compile and install a new one + yum install -y xz + wget https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz + unxz libxml2-2.9.14.tar.xz + tar -xf libxml2-2.9.14.tar + cd libxml2-2.9.14 && ./configure --prefix=$PREFIX --without-python && make && make install + cd .. + fi + ;; + 8*) + #they forgot to package libexttextcat-devel? grab one manually: + wget https://github.com/proycon/LaMachine/raw/master/deps/centos8/libexttextcat-devel-3.4.5-2.el8.x86_64.rpm + yum install -y libexttextcat-devel-3.4.5-2.el8.x86_64.rpm + ;; + esac fi PWD="$(pwd)" @@ -77,5 +82,4 @@ for PACKAGE in LanguageMachines/ticcutils LanguageMachines/libfolia LanguageMach done cd $PWD [ -n "$BUILDDIR" ] && rm -Rf "$BUILDDIR" - echo "Dependencies installed" >&2 diff --git a/setup.py b/setup.py index 81c086c..0c62463 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name = 'python-frog', - version = '0.6.10', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.11', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 7cff8d36fc9f9ea288d4a09bf9312e7762eb1bc3 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 17 Dec 2024 12:54:47 +0100 Subject: [PATCH 63/75] ci: no aarch64 for musllinux yet --- .github/workflows/wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 992f55e..8112e16 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -13,7 +13,6 @@ jobs: buildplat: - { runs_on: ubuntu-20.04, sys: manylinux, arch: x86_64, benv: "" } - { runs_on: ubuntu-20.04, sys: musllinux, arch: x86_64, benv: "" } - - { runs_on: ubuntu-20.04, sys: musllinux, arch: aarch64, benv: "" } - { runs_on: macos-14, sys: macosx, arch: arm64, benv: "14.0" } python: - { cp: "cp38", rel: "3.8" } @@ -59,7 +58,7 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* - CIBW_ARCHS_LINUX: "x86_64 aarch64" + CIBW_ARCHS_LINUX: "x86_64" CIBW_BEFORE_ALL_LINUX: > apk add build-base git autoconf-archive autoconf automake libtool bzip2-dev icu-dev libxml2-dev libexttextcat-dev libtool rsync && mkdir -p /usr/local/share/aclocal/ && rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ && From 0dea00c555d31fe9141e7f9c03fc419692ce60e7 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Tue, 17 Dec 2024 13:07:16 +0100 Subject: [PATCH 64/75] use c++17 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0c62463..8002bf3 100755 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ include_dirs=includedirs, library_dirs=libdirs, libraries=['frog','ucto','folia'], - extra_compile_args=['--std=c++0x'] + extra_options) + extra_compile_args=['--std=c++17'] + extra_options) ], compiler_directives={"language_level": "3"} ) From b411da925d1e58922a2339f8a93e78be6d4a649f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 20 Mar 2025 17:20:36 +0100 Subject: [PATCH 65/75] fix numthreads parsing, expect int value Not ideal though (int to str in cython and back to int on the C++ side) Ref: https://github.com/proycon/python-frog/issues/5 --- frog_wrapper.pyx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frog_wrapper.pyx b/frog_wrapper.pyx index fc5ac48..8da1948 100644 --- a/frog_wrapper.pyx +++ b/frog_wrapper.pyx @@ -127,7 +127,10 @@ cdef class FrogOptions: else: self.capi.insert(b"id", value.encode('utf-8')) elif key.lower() in ('numthreads','threads'): - self.capi.insert(b"threads",value) + if isinstance(value, int): + self.capi.insert(b"threads",str(value).encode('utf-8')) + else: + self.capi.insert(b"threads",value) else: if key == 'x': self.shadow['xmlin'] = True From 1b1cb85fa118463a3d8dd5ae175c71ab391a126a Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 20 Mar 2025 17:22:35 +0100 Subject: [PATCH 66/75] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8002bf3..c091c4f 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name = 'python-frog', - version = '0.6.11', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.12', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 345b52c7e2dae72953571e90b8e63d5c61bf1651 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 2 Feb 2026 10:25:03 +0100 Subject: [PATCH 67/75] ci: added python 3.14 to wheel builder --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8112e16..b6f230e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,6 +21,7 @@ jobs: - { cp: "cp311", rel: "3.11" } - { cp: "cp312", rel: "3.12" } - { cp: "cp313", rel: "3.13" } + - { cp: "cp314", rel: "3.14" } steps: - uses: actions/checkout@v4.1.1 From ce9a6904658d87700fd37cf696a840b86173743f Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 2 Feb 2026 15:25:09 +0100 Subject: [PATCH 68/75] ci: wheels: ubuntu 20.04 is deprecated on github actions, upgrading --- .github/workflows/wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b6f230e..3ab0821 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -11,8 +11,8 @@ jobs: strategy: matrix: buildplat: - - { runs_on: ubuntu-20.04, sys: manylinux, arch: x86_64, benv: "" } - - { runs_on: ubuntu-20.04, sys: musllinux, arch: x86_64, benv: "" } + - { runs_on: ubuntu-22.04, sys: manylinux, arch: x86_64, benv: "" } + - { runs_on: ubuntu-22.04, sys: musllinux, arch: x86_64, benv: "" } - { runs_on: macos-14, sys: macosx, arch: arm64, benv: "14.0" } python: - { cp: "cp38", rel: "3.8" } From f30a543ce83bca84ac80c0b6088cc61bb9784ce3 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 2 Feb 2026 15:37:02 +0100 Subject: [PATCH 69/75] ci: wheels: no python 3.14 on alpine yet --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3ab0821..a0b3a7e 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -55,7 +55,7 @@ jobs: CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - name: Build wheels (Linux musl) - if: ${{ matrix.buildplat.sys == 'musllinux' }} + if: ${{ matrix.buildplat.sys == 'musllinux' && matrix.python.cp != 'cp314' }} run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* From 6201c88aa29df766efd938787613ea781394ffec Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 2 Feb 2026 15:54:42 +0100 Subject: [PATCH 70/75] setup.py: license classifier is deprecated, removing --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c091c4f..feb491c 100755 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), - license = "GPLv3", + license = "GPL-3.0-only", keywords = "nlp computational_linguistics dutch pos lemmatizer", url = "https://github.com/proycon/python-frog", ext_modules = extensions, @@ -86,6 +86,5 @@ "Operating System :: POSIX", "Intended Audience :: Developers", "Intended Audience :: Science/Research", - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", ], ) From 4d80c45ecd185bef3f5df70febfda1848e7c5868 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Mon, 17 Aug 2026 14:51:51 +0200 Subject: [PATCH 71/75] Revert "ci: wheels: no python 3.14 on alpine yet" This reverts commit f30a543ce83bca84ac80c0b6088cc61bb9784ce3. --- .github/workflows/wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a0b3a7e..3ab0821 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -55,7 +55,7 @@ jobs: CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - name: Build wheels (Linux musl) - if: ${{ matrix.buildplat.sys == 'musllinux' && matrix.python.cp != 'cp314' }} + if: ${{ matrix.buildplat.sys == 'musllinux' }} run: python -m cibuildwheel --output-dir wheelhouse env: CIBW_BUILD: ${{ matrix.python.cp }}-${{ matrix.buildplat.sys }}* From ca582a8feadd326b53be5d8be5c83a9860d7db43 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 20 Aug 2026 13:38:08 +0200 Subject: [PATCH 72/75] ci: updated musllinux base image --- .github/workflows/wheels.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3ab0821..e4c81b5 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -19,9 +19,9 @@ jobs: - { cp: "cp39", rel: "3.9" } - { cp: "cp310", rel: "3.10" } - { cp: "cp311", rel: "3.11" } - - { cp: "cp312", rel: "3.12" } - - { cp: "cp313", rel: "3.13" } - - { cp: "cp314", rel: "3.14" } + - { cp: "cp312", rel: "3.12" } + - { cp: "cp313", rel: "3.13" } + - { cp: "cp314", rel: "3.14" } steps: - uses: actions/checkout@v4.1.1 @@ -64,8 +64,8 @@ jobs: apk add build-base git autoconf-archive autoconf automake libtool bzip2-dev icu-dev libxml2-dev libexttextcat-dev libtool rsync && mkdir -p /usr/local/share/aclocal/ && rsync -av --ignore-existing /usr/share/aclocal/*.m4 /usr/local/share/aclocal/ && ./build-deps.sh - CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/musllinux_1_1_x86_64 - CIBW_MUSLLINUX_AARCH64_IMAGE: quay.io/pypa/musllinux_1_1_aarch64 + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/musllinux_1_2_x86_64 + CIBW_MUSLLINUX_AARCH64_IMAGE: quay.io/pypa/musllinux_1_2_aarch64 CIBW_SKIP: "*-win* *-manylinux_i686 pp*" - name: Build wheels (macOS) From 4fdc934a765a4c694c819de248947b3123a659e8 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 20 Aug 2026 13:38:59 +0200 Subject: [PATCH 73/75] ci: drop python 3.8 support --- .github/workflows/wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e4c81b5..4807a3a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -15,7 +15,6 @@ jobs: - { runs_on: ubuntu-22.04, sys: musllinux, arch: x86_64, benv: "" } - { runs_on: macos-14, sys: macosx, arch: arm64, benv: "14.0" } python: - - { cp: "cp38", rel: "3.8" } - { cp: "cp39", rel: "3.9" } - { cp: "cp310", rel: "3.10" } - { cp: "cp311", rel: "3.11" } From 3e226802cc9075d7fadb8576bb0e5d3f7be1b557 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 20 Aug 2026 13:39:52 +0200 Subject: [PATCH 74/75] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index feb491c..07826c0 100755 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ setup( name = 'python-frog', - version = '0.6.12', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx + version = '0.6.13', #also ensure UCTODATAVERSION and FROGDATAVERSION are good in frog_wrapper.pyx author = "Maarten van Gompel", author_email = "proycon@anaproy.nl", description = ("Python binding to Frog, an NLP suite for Dutch doing part-of-speech tagging, lemmatisation, morphological analysis, named-entity recognition, shallow parsing, and dependency parsing."), From 8a59954949f7517e65e5788e0d9da1056f321707 Mon Sep 17 00:00:00 2001 From: Maarten van Gompel Date: Thu, 20 Aug 2026 13:50:00 +0200 Subject: [PATCH 75/75] ci: fix for macos builds --- .github/workflows/wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 4807a3a..2df8e3c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -75,6 +75,7 @@ jobs: CIBW_ARCHS: ${{ matrix.buildplat.arch }} CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat.benv }}" CIBW_BEFORE_ALL_MACOS: > + brew trust fbkarsdorp/lamachine && brew tap fbkarsdorp/homebrew-lamachine && brew install frog