I'm maintaining a python package which includes a cython-based c extention. The source code is on github: .
Here is my setup.py:
import os from setuptools import setup, find_packages from distutils.core import Extension try: import numpy except ImportError: # We do not have numpy installed os.system("pip install numpy") try: from Cython.Build import cythonize except ImportError: # We do not have Cython installed os.system("pip install Cython") import numpy from Cython.Build import cythonize __version__ = "0.1.0-b3" exts = [Extension(name='vltools.nms.nms_ext', sources=["vltools/nms/nms_ext.pyx"], include_dirs=[numpy.get_include()]) ] setup(name='vltools', version=__version__, description='vision and learning tools', url=' author_email='[email protected]', license='MIT', packages=find_packages(), ext_modules=cythonize(exts), zip_safe=False, data_files=[("data", ["data/imagenet1000_clsidx_to_labels.txt"])] ) When building locally with python setup build && python setup.py install, everything goes smoothly.
However, when I'm trying to create a source distribution with python setup.py sdist and then install from the generated dist/vltools-0.1.0b3.tar.gz it run into an error:
ValueError: 'vltools/nms/nms_ext.pyx' doesn't match any files In my understanding, the one actually required for installation is nms_ext.c which is indeed inside the generated dist/vltools-0.1.0b3.tar.gz.
However, in my setup.py it's "nms_ext.pyx" in the sources:
exts = [Extension(name='vltools.nms.nms_ext', sources=["vltools/nms/nms_ext.pyx"], include_dirs=[numpy.get_include()]) ] So what should I do when creating a source distribution with python setup.py sdist?
2 Answers
There are several things to fix or improve in your setup script.
Declaring build and runtime dependencies
Instead of running pip install {numpy,cython} where a lot of things can go wrong, a proper way of declaring dependencies is to pass them to setup() in the setup_requires/install_requires args. Cython is required only at the build stage, but not when vltools is installed and imported - it will be added to setup_requires. numpy is required to both build and run the package, so it is included in both lists:
setup( ..., setup_requires=["cython", "numpy"], install_requires=["numpy"], ) This has the advantage that Cython won't be installed. It will be downloaded, used for building and then removed.
You can (and should) extend the install_requires list with other packages vltools requires, for example scipy, pillow etc.
Deferring Cython and numpy imports
To ensure the package is installable on a system that has neither Cython nor numpy installed beforehand, we have to defer their imports. I won't go into much detail here; check out Add numpy.get_include() argument to setuptools without preinstalled numpy for more details. In the custom impl of the build command below, I defer both cythonizing and extending extension headers with numpy includes:
class build(build_orig): def finalize_options(self): super().finalize_options() __builtins__.__NUMPY_SETUP__ = False import numpy for extension in self.distribution.ext_modules: extension.include_dirs.append(numpy.get_include()) from Cython.Build import cythonize self.distribution.ext_modules = cythonize(self.distribution.ext_modules, language_level=3) Register the custom command impl via cmdclass:
setup( ..., cmdclass={"build": build}, ) Packaging nms.h
Currently, the nms.h is not being added to your source dist, making the installation impossible. This is easily fixed by including it via e.g. package_data:
setup( ..., package_data={"vltools.nms": ["nms.h"]}, ) Now you also should add the parent dir of nms.h to include_dirs:
exts = [Extension(name='vltools.nms.nms_ext', sources=["vltools/nms/_nms_ext.c", "vltools/nms/nms_ext.pyx"], include_dirs=["vltools/nms"])] Full setup script
import os from setuptools import setup, find_packages from setuptools import Extension from distutils.command.build import build as build_orig __version__ = "0.1.0b3" exts = [Extension(name='vltools.nms.nms_ext', sources=["vltools/nms/_nms_ext.c", "vltools/nms/nms_ext.pyx"], include_dirs=["vltools/nms"])] class build(build_orig): def finalize_options(self): super().finalize_options() __builtins__.__NUMPY_SETUP__ = False import numpy for extension in self.distribution.ext_modules: extension.include_dirs.append(numpy.get_include()) from Cython.Build import cythonize self.distribution.ext_modules = cythonize(self.distribution.ext_modules, language_level=3) setup(name='vltools', version=__version__, description='vision and learning tools', url=' author_email='[email protected]', license='MIT', packages=find_packages(), ext_modules=exts, setup_requires=["cython", "numpy"], install_requires=["numpy"], zip_safe=False, data_files=[("data", ["data/imagenet1000_clsidx_to_labels.txt"])], package_data={"vltools.nms": ["nms.h"]}, cmdclass={"build": build}, ) P.S. Two years have passed and here we meet again ;-)
3As @hoefling suggested, I've updated my setup.py.
However, there is still an error when installing from .tar.gz file that it cannot find 'vltools/nms/nms_ext.pyx'.
setup.py hoefling posted:
import os from setuptools import setup, find_packages from setuptools import Extension from distutils.command.build import build as build_orig __version__ = "0.1.0b3" exts = [Extension(name='vltools.nms.nms_ext', sources=["vltools/nms/_nms_ext.c", "vltools/nms/nms_ext.pyx"], include_dirs=["vltools/nms"])] class build(build_orig): def finalize_options(self): super().finalize_options() __builtins__.__NUMPY_SETUP__ = False import numpy for extension in self.distribution.ext_modules: extension.include_dirs.append(numpy.get_include()) from Cython.Build import cythonize self.distribution.ext_modules = cythonize(self.distribution.ext_modules, language_level=3) setup(name='vltools', version=__version__, description='vision and learning tools', url=' author_email='[email protected]', license='MIT', packages=find_packages(), ext_modules=exts, setup_requires=["cython", "numpy"], install_requires=["numpy"], zip_safe=False, data_files=[("data", ["data/imagenet1000_clsidx_to_labels.txt"])], package_data={"vltools.nms": ["nms.h"]}, cmdclass={"build": build}, ) When installing with python setup.py sdist && cd dist && pip install vltools-0.1.0b3.tar.gz, there is an error saying:
ValueError: 'vltools/nms/nms_ext.pyx' doesn't match any files Becase the pyx file was not packed into tar.gz.
To have 'vltools/nms/nms_ext.pyx' in the tar.gz file, I add it into the package_data list:
package_data={"vltools.nms": ["nms.h", "nms_ext.pyx"]} Finally, my full setup.py is:
import os from setuptools import setup, find_packages from setuptools import Extension from distutils.command.build import build as build_orig __version__ = "0.1.0b3" exts = [Extension(name='vltools.nms.nms_ext', sources=["vltools/nms/_nms_ext.c", "vltools/nms/nms_ext.pyx"], include_dirs=["vltools/nms"])] class build(build_orig): def finalize_options(self): super().finalize_options() __builtins__.__NUMPY_SETUP__ = False import numpy for extension in self.distribution.ext_modules: extension.include_dirs.append(numpy.get_include()) from Cython.Build import cythonize self.distribution.ext_modules = cythonize(self.distribution.ext_modules, language_level=3) setup(name='vltools', version=__version__, description='vision and learning tools', url=' author_email='[email protected]', license='MIT', packages=find_packages(), ext_modules=exts, setup_requires=["cython", "numpy"], install_requires=["numpy"], zip_safe=False, data_files=[("data", ["data/imagenet1000_clsidx_to_labels.txt"])], package_data={"vltools.nms": ["nms.h", "nms_ext.pyx"]}, cmdclass={"build": build}, ) 
