Basically, make python builds work as expected

bindings/python/MANIFEST.in

@@ -1,4 +1,4 @@
 recursive-include src *
 recursive-include prebuilt *
 include LICENSE.TXT
-include README
+include README.TXT

bindings/python/Makefile

@@ -2,7 +2,7 @@
 
 OBJDIR = ./build
 
-.PHONY: gen_const install install3 clean
+.PHONY: gen_const install install3 clean sdist sdist3 bdist bdist3
 
 gen_const:
 	cd .. && python const_generator.py python
@@ -32,7 +32,6 @@ sdist:
 	rm -rf src/ dist/
 	rm -rf prebuilt/win64/unicorn.dll
 	rm -rf prebuilt/win32/unicorn.dll
-	cp README.pypi-src README
 	cp PKG-INFO.src PKG-INFO
 	python setup.py sdist register upload
 
@@ -41,15 +40,29 @@ sdist3:
 	rm -rf src/ dist/
 	rm -rf prebuilt/win64/unicorn.dll
 	rm -rf prebuilt/win32/unicorn.dll
-	cp README.pypi-src README
 	cp PKG-INFO.src PKG-INFO
 	python3 setup.py sdist register upload
 
+# build & upload PyPi package with precompiled core
+bdist:
+	rm -rf src/ dist/
+	rm -rf prebuilt/win64/unicorn.dll
+	rm -rf prebuilt/win32/unicorn.dll
+	cp PKG-INFO.src PKG-INFO
+	python setup.py bdist_wheel register upload
+
+# build & upload PyPi package with precompiled core
+bdist3:
+	rm -rf src/ dist/
+	rm -rf prebuilt/win64/unicorn.dll
+	rm -rf prebuilt/win32/unicorn.dll
+	cp PKG-INFO.src PKG-INFO
+	python3 setup.py bdist_wheel register upload
+
 # build & upload PyPi package with prebuilt core
 # NOTE: be sure to have precompiled core under prebuilt/win*/ beforehand
 sdist_win:
 	rm -rf src/ dist/
-	cp README.pypi-win README
 	cp PKG-INFO.win PKG-INFO
 	python setup.py sdist register upload
 
@@ -57,15 +70,16 @@ sdist_win:
 # NOTE: be sure to have precompiled core under prebuilt/win*/ beforehand
 sdist3_win:
 	rm -rf src/ dist/
-	cp README.pypi-win README
 	cp PKG-INFO.win PKG-INFO
 	python3 setup.py sdist register upload
 
 clean:
-	rm -rf $(OBJDIR) src/ dist/ README
+	rm -rf $(OBJDIR) src/ dist/ MANIFEST
-	rm -f unicorn/*.so
 	rm -rf prebuilt/win64/unicorn.dll
 	rm -rf prebuilt/win32/unicorn.dll
+	rm -rf unicorn/lib unicorn/include
+	rm -rf unicorn/*.pyc
+	rm -rf unicorn.egg-info
 
 
 SAMPLES = sample_arm.py sample_arm64.py sample_mips.py

bindings/python/README.TXT

@@ -1,26 +1,27 @@
-This documentation explains how to install Python binding for Unicorn
+This documentation explains how to install the python binding for Unicorn
 from source.
 
+1. Installing on Linux:
 
-0. Install the core engine as dependency
+	$ sudo python setup.py install
 
-   Follow README in the root directory to compile & install the core.
+	This will build the core C library, package it with the python bindings, 
-
+	and install it to your system.
-   On *nix, this can simply be done by (project root directory):
-
-        $ sudo ./make.sh install
 
 
-1. To install pure Python binding on *nix, run the command below in the Python bindings directory:
+2. Installing on Windows:
 
-		$ sudo make install
+	Run the following command in command prompt:
 
-  To install Python3 binding package, run the command below:
+		C:\> C:\location_to_python\python.exe setup.py install
-  (Note: this requires python3 installed in your machine)
 
-		$ sudo make install3
+	Next, copy all the DLL files from the 'Core engine for Windows' package available
+	on the Unicorn download page and paste it in the path:
+
+		C:\location_to_python\Lib\site-packages\unicorn\
 
 
+3. Sample code
 
 	This directory contains some sample code to show how to use Unicorn API.
 
@@ -33,15 +34,3 @@ This directory contains some sample code to show how to use Unicorn API.
 
 	- sample_network_auditing.py
 	  This shows how to analyze & interpret Linux shellcode.
-
-
-2. To install Python binding on Windows:
-
-	Run the following command in command prompt:
-
-		C:\> C:\location_to_python\python.exe setup.py install
-
-	Next, copy all the DLL files from the 'Core engine for Windows' package available
-	on the same Unicorn download page and paste it in the path:
-
-		C:\location_to_python\Lib\site-packages\unicorn\

bindings/python/setup.py

@@ -3,17 +3,14 @@
 
 import glob
 import os
-import platform
 import shutil
-import stat
 import sys
 
 from distutils import log
-from distutils import dir_util
-from distutils.command.build_clib import build_clib
-from distutils.command.sdist import sdist
 from distutils.core import setup
-from distutils.sysconfig import get_python_lib
+from distutils.command.build import build
+from distutils.command.sdist import sdist
+from setuptools.command.bdist_egg import bdist_egg
 
 # prebuilt libraries for Windows - for sdist
 PATH_LIB64 = "prebuilt/win64/unicorn.dll"
@@ -24,24 +21,33 @@ PKG_NAME = 'unicorn'
 if os.path.exists(PATH_LIB64) and os.path.exists(PATH_LIB32):
     PKG_NAME = 'unicorn-windows'
 
-VERSION = '1.0'
 SYSTEM = sys.platform
-
+VERSION = '1.0'
-# virtualenv breaks import, but get_python_lib() will work.
-SITE_PACKAGES = os.path.join(get_python_lib(), "unicorn")
-if "--user" in sys.argv:
-    try:
-        from site import getusersitepackages
-        SITE_PACKAGES = os.path.join(getusersitepackages(), "unicorn")
-    except ImportError:
-        pass
-
-
-SETUP_DATA_FILES = []
 
 # adapted from commit e504b81 of Nguyen Tan Cong
 # Reference: https://docs.python.org/2/library/platform.html#cross-platform
-is_64bits = sys.maxsize > 2**32
+IS_64BITS = sys.maxsize > 2**32
+
+# are we building from the repository or from a source distribution?
+ROOT_DIR = os.path.dirname(os.path.realpath(__file__))
+LIBS_DIR = os.path.join(ROOT_DIR, 'unicorn', 'lib')
+HEADERS_DIR = os.path.join(ROOT_DIR, 'unicorn', 'include')
+SRC_DIR = os.path.join(ROOT_DIR, 'src')
+BUILD_DIR = SRC_DIR if os.path.exists(SRC_DIR) else os.path.join(ROOT_DIR, '../..')
+
+if SYSTEM == 'darwin':
+    LIBRARY_FILE = "libunicorn.1.dylib"
+    STATIC_LIBRARY_FILE = 'libunicorn.a'
+elif SYSTEM in ('win32', 'cygwin'):
+    LIBRARY_FILE = "unicorn.dll"
+    STATIC_LIBRARY_FILE = None
+else:
+    LIBRARY_FILE = "libunicorn.so.1"
+    STATIC_LIBRARY_FILE = 'libunicorn.a'
+
+def clean_bins():
+    shutil.rmtree(LIBS_DIR, ignore_errors=True)
+    shutil.rmtree(HEADERS_DIR, ignore_errors=True)
 
 def copy_sources():
     """Copy the C sources into the source directory.
@@ -50,106 +56,117 @@ def copy_sources():
     """
     src = []
 
-    try:
+    os.system('make -C %s clean' % os.path.join(ROOT_DIR, '../..'))
-        dir_util.remove_tree("src/")
+    shutil.rmtree(SRC_DIR, ignore_errors=True)
-    except (IOError, OSError):
+    os.mkdir(SRC_DIR)
-        pass
 
-    dir_util.copy_tree("../../arch", "src/arch/")
+    shutil.copytree(os.path.join(ROOT_DIR, '../../qemu'), os.path.join(SRC_DIR, 'qemu/'))
-    dir_util.copy_tree("../../include", "src/include/")
+    shutil.copytree(os.path.join(ROOT_DIR, '../../include'), os.path.join(SRC_DIR, 'include/'))
+    # make -> configure -> clean -> clean tests fails unless tests is present
+    shutil.copytree(os.path.join(ROOT_DIR, '../../tests'), os.path.join(SRC_DIR, 'tests/'))
+    # remove site-specific configuration file
+    os.remove(os.path.join(SRC_DIR, 'qemu/config-host.mak'))
 
-    src.extend(glob.glob("../../*.[ch]"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.[ch]")))
-    src.extend(glob.glob("../../*.mk"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.mk")))
 
-    src.extend(glob.glob("../../Makefile"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../Makefile")))
-    src.extend(glob.glob("../../LICENSE*"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../LICENSE*")))
-    src.extend(glob.glob("../../README.md"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../README.md")))
-    src.extend(glob.glob("../../*.TXT"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.TXT")))
-    src.extend(glob.glob("../../RELEASE_NOTES"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../RELEASE_NOTES")))
-    src.extend(glob.glob("../../make.sh"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../make.sh")))
-    src.extend(glob.glob("../../CMakeLists.txt"))
+    src.extend(glob.glob(os.path.join(ROOT_DIR, "../../CMakeLists.txt")))
 
     for filename in src:
-        outpath = os.path.join("./src/", os.path.basename(filename))
+        outpath = os.path.join(SRC_DIR, os.path.basename(filename))
         log.info("%s -> %s" % (filename, outpath))
         shutil.copy(filename, outpath)
 
+def build_libraries():
+    """
+    Prepare the unicorn directory for a binary distribution or installation.
+    Builds shared libraries and copies header files.
 
-class custom_sdist(sdist):
+    Will use a src/ dir if one exists in the current directory, otherwise assumes it's in the repo
-    """Reshuffle files for distribution."""
+    """
+    cwd = os.getcwd()
+    clean_bins()
+    os.mkdir(HEADERS_DIR)
+    os.mkdir(LIBS_DIR)
 
-    def run(self):
+    # copy public headers
-        # if prebuilt libraries are existent, then do not copy source
+    shutil.copytree(os.path.join(BUILD_DIR, 'include', 'unicorn'), os.path.join(HEADERS_DIR, 'unicorn'))
-        if os.path.exists(PATH_LIB64) and os.path.exists(PATH_LIB32):
-            return sdist.run(self)
-        copy_sources()
-        return sdist.run(self)
 
-
-class custom_build_clib(build_clib):
-    """Customized build_clib command."""
-
-    def run(self):
-        log.info('running custom_build_clib')
-        build_clib.run(self)
-
-    def finalize_options(self):
-        # We want build-clib to default to build-lib as defined by the "build"
-        # command.  This is so the compiled library will be put in the right
-        # place along side the python code.
-        self.set_undefined_options('build',
-                                   ('build_lib', 'build_clib'),
-                                   ('build_temp', 'build_temp'),
-                                   ('compiler', 'compiler'),
-                                   ('debug', 'debug'),
-                                   ('force', 'force'))
-
-        build_clib.finalize_options(self)
-
-    def build_libraries(self, libraries):
-        if SYSTEM in ("win32", "cygwin"):
     # if Windows prebuilt library is available, then include it
-            if is_64bits and os.path.exists(PATH_LIB64):
+    if SYSTEM in ("win32", "cygwin"):
-                SETUP_DATA_FILES.append(PATH_LIB64)
+        if IS_64BITS and os.path.exists(PATH_LIB64):
+            shutil.copy(PATH_LIB64, LIBS_DIR)
             return
         elif os.path.exists(PATH_LIB32):
-                SETUP_DATA_FILES.append(PATH_LIB32)
+            shutil.copy(PATH_LIB32, LIBS_DIR)
             return
 
-        # build library from source if src/ is existent
+    # otherwise, build!!
-        if not os.path.exists('src'):
+    os.chdir(BUILD_DIR)
-            return
 
-        try:
+    # platform description refs at https://docs.python.org/2/library/sys.html#sys.platform
-            for (lib_name, build_info) in libraries:
-                log.info("building '%s' library", lib_name)
-
-                os.chdir("src")
-
-                # platform description refers at https://docs.python.org/2/library/sys.html#sys.platform
     if SYSTEM == "cygwin":
-                    os.chmod("make.sh", stat.S_IREAD|stat.S_IEXEC)
+        if IS_64BITS:
-                    if is_64bits:
             os.system("UNICORN_BUILD_CORE_ONLY=yes ./make.sh cygwin-mingw64")
         else:
             os.system("UNICORN_BUILD_CORE_ONLY=yes ./make.sh cygwin-mingw32")
-                    SETUP_DATA_FILES.append("src/unicorn.dll")
     else:   # Unix
-                    os.chmod("make.sh", stat.S_IREAD|stat.S_IEXEC)
         os.system("UNICORN_BUILD_CORE_ONLY=yes ./make.sh")
-                    if SYSTEM == "darwin":
-                        SETUP_DATA_FILES.append("src/libunicorn.dylib")
-                    else:   # Non-OSX
-                        SETUP_DATA_FILES.append("src/libunicorn.so")
 
-                os.chdir("..")
+    shutil.copy(LIBRARY_FILE, LIBS_DIR)
-        except:
+    if STATIC_LIBRARY_FILE: shutil.copy(STATIC_LIBRARY_FILE, LIBS_DIR)
-            pass
+    os.chdir(cwd)
+    if SYSTEM == "linux2": os.symlink(LIBRARY_FILE, os.path.join(LIBS_DIR, 'libunicorn.so'))
 
 
+class custom_sdist(sdist):
+    def run(self):
+        clean_bins()
+
+        # if prebuilt libraries are existent, then do not copy source
+        if not os.path.exists(PATH_LIB64) or not os.path.exists(PATH_LIB32):
+            copy_sources()
+        return sdist.run(self)
+
+class custom_build(build):
+    def run(self):
+        log.info("Building C extensions")
+        build_libraries()
+        return build.run(self)
+
+class custom_bdist_egg(bdist_egg):
+    def run(self):
+        self.run_command('build')
+        return bdist_egg.run(self)
+
 def dummy_src():
     return []
 
+cmdclass = {}
+cmdclass['build'] = custom_build
+cmdclass['sdist'] = custom_sdist
+cmdclass['bdist_egg'] = custom_bdist_egg
+
+try:
+    from setuptools.command.develop import develop
+    class custom_develop(develop):
+        def run(self):
+            log.info("Building C extensions")
+            build_libraries()
+            return develop.run(self)
+
+    cmdclass['develop'] = custom_develop
+except ImportError:
+    print "Proper 'develop' support unavailable."
+
+def join_all(src, files):
+    return tuple(os.path.join(src, f) for f in files)
 
 setup(
     provides=['unicorn'],
@@ -166,17 +183,16 @@ setup(
         'Programming Language :: Python :: 3',
     ],
     requires=['ctypes'],
-    cmdclass=dict(
+    cmdclass=cmdclass,
-        build_clib=custom_build_clib,
-        sdist=custom_sdist,
-    ),
-
     libraries=[(
         'unicorn', dict(
             package='unicorn',
             sources=dummy_src()
         ),
     )],
-
+    zip_safe=True,
-    data_files=[(SITE_PACKAGES, SETUP_DATA_FILES)],
+    include_package_data=True,
+    package_data={
+        'unicorn': ['lib/*', 'include/unicorn/*']
+    }
 )

bindings/python/unicorn/unicorn.py

@@ -3,9 +3,9 @@
 import ctypes
 import ctypes.util
 import distutils.sysconfig
+import pkg_resources
 import inspect
 import os.path
-import platform
 import sys
 
 from . import x86_const, unicorn_const as uc
@@ -17,12 +17,12 @@ _python2 = sys.version_info[0] < 3
 if _python2:
     range = xrange
 
-_lib_path = os.path.split(__file__)[0]
+if sys.platform == 'darwin':
-_all_libs = (
+    _lib = "libunicorn.1.dylib"
-    "unicorn.dll",
+elif sys.platform in ('win32', 'cygwin'):
-    "libunicorn.so",
+    _lib = "unicorn.dll"
-    "libunicorn.dylib",
+else:
-)
+    _lib = "libunicorn.so.1"
 
 # Windows DLL in dependency order
 _all_windows_dlls = (
@@ -33,71 +33,44 @@ _all_windows_dlls = (
     "libintl-8.dll",
     "libglib-2.0-0.dll",
 )
-_found = False
 
-for _lib in _all_libs:
+def _load_win_support(path):
+    for dll in _all_windows_dlls:
+        lib_file = os.path.join(path, dll)
+        if os.path.exists(lib_file):
+            ctypes.cdll.LoadLibrary(lib_file)
+
+def _load_lib(path):
     try:
-        if _lib == "unicorn.dll":
+        if sys.platform in ('win32', 'cygwin'):
-            for dll in _all_windows_dlls:    # load all the rest DLLs first
+            _load_win_support(path)
-                _lib_file = os.path.join(_lib_path, dll)
+
-                if os.path.exists(_lib_file):
+        lib_file = os.path.join(path, _lib)
-                    ctypes.cdll.LoadLibrary(_lib_file)
+        return ctypes.cdll.LoadLibrary(lib_file)
-        _lib_file = os.path.join(_lib_path, _lib)
-        _uc = ctypes.cdll.LoadLibrary(_lib_file)
-        _found = True
-        break
     except OSError:
-        pass
+        return None
 
-if not _found:
+_uc = None
-    # try loading from default paths
-    for _lib in _all_libs:
-        try:
-            _uc = ctypes.cdll.LoadLibrary(_lib)
-            _found = True
-            break
-        except OSError:
-            pass
 
-if not _found:
+# Loading attempts, in order
-    # last try: loading from python lib directory
+# - pkg_resources can get us the path to the local libraries
-    _lib_path = distutils.sysconfig.get_python_lib()
+# - we can get the path to the local libraries by parsing our filename
-    for _lib in _all_libs:
+# - global load
-        try:
+# - python's lib directory
-            if _lib == "unicorn.dll":
+# - last-gasp attempt at some hardcoded paths on darwin and linux
-                for dll in _all_windows_dlls:    # load all the rest DLLs first
-                    _lib_file = os.path.join(_lib_path, "unicorn", dll)
-                    if os.path.exists(_lib_file):
-                        ctypes.cdll.LoadLibrary(_lib_file)
-            _lib_file = os.path.join(_lib_path, "unicorn", _lib)
-            _uc = ctypes.cdll.LoadLibrary(_lib_file)
-            _found = True
-            break
-        except OSError:
-            pass
 
-if not _found:
+_path_list = [pkg_resources.resource_filename(__name__, 'lib'),
-    # Attempt Darwin specific load (10.11 specific),
+              os.path.join(os.path.split(__file__)[0], 'lib'),
-    # since LD_LIBRARY_PATH is not guaranteed to exist
+              '',
-    if platform.system() == "Darwin":
+              distutils.sysconfig.get_python_lib(),
-        _lib_path = "/usr/local/lib/"
+              "/usr/local/lib/" if sys.platform == 'darwin' else '/usr/lib64']
-    elif platform.system() == "Linux":
-        _lib_path = "/usr/lib64/"
 
-    for _lib in _all_libs:
+for _path in _path_list:
-        try:
+    _uc = _load_lib(_path)
-            _lib_file = os.path.join(_lib_path, _lib)
+    if _uc is not None: break
-            # print "Trying to load:", _lib_file
+else:
-            _uc = ctypes.cdll.LoadLibrary(_lib_file)
-            _found = True
-            break
-        except OSError:
-            pass
-
-if not _found:
     raise ImportError("ERROR: fail to load the dynamic library.")
 
-
 __version__ = "%s.%s" % (uc.UC_API_MAJOR, uc.UC_API_MINOR)
 
 # setup all the function prototype