diff --git a/README.md b/README.md index 6e530bb4e1..c20715c5b2 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Broad goals of the revamp project include: - (WIP) Support python3 (recipe exists but crashes on android) - (WIP) Support some kind of binary distribution, including on windows (semi-implemented, just needs finishing) - ✓ Be a standalone Pypi module (not on pypi yet but setup.py works) -- Support multiple architectures +- ✓ Support multiple architectures (full multiarch builds not complete, but arm and x86 with different config both work now) We are currently working to stabilise all parts of the toolchain and add more features. Support for pygame-based APKs is almost feature diff --git a/doc/source/commands.rst b/doc/source/commands.rst index 684591002f..57d11994b1 100644 --- a/doc/source/commands.rst +++ b/doc/source/commands.rst @@ -70,6 +70,11 @@ supply those that you need. ``--force_build BOOL`` Whether the distribution must be compiled from scratch. +``--arch`` + The architecture to build for. Currently only one architecture can be + targeted at a time, and a given distribution can only include one architecture. + + .. note:: These options are preliminary. Others will include toggles for allowing downloads, and setting additional directories from which to load user dists. diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 8f890b8acb..a0ccc66bb2 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -254,10 +254,10 @@ correct and try to continue the build. Configuration file ~~~~~~~~~~~~~~~~~~ -python-for-android look on the current directory if there is a `.p4a` -configuration file. If it found it, it adds all the lines as options -to the command line. For example, you can put the options you would -always write such as: +python-for-android checks in the current directory for a configuration +file named ``.p4a``. If found, it adds all the lines as options to the +command line. For example, you can add the options you would always +include such as: --dist_name my_example --android_api 19 diff --git a/pythonforandroid/bootstraps/empty/__init__.py b/pythonforandroid/bootstraps/empty/__init__.py index 53f7334166..10137a78e8 100644 --- a/pythonforandroid/bootstraps/empty/__init__.py +++ b/pythonforandroid/bootstraps/empty/__init__.py @@ -1,4 +1,4 @@ -from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchAndroid, logger, info_main, which +from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchARM, logger, info_main, which from os.path import join, exists from os import walk import glob diff --git a/pythonforandroid/bootstraps/pygame/__init__.py b/pythonforandroid/bootstraps/pygame/__init__.py index a0ddaac626..be2589364e 100644 --- a/pythonforandroid/bootstraps/pygame/__init__.py +++ b/pythonforandroid/bootstraps/pygame/__init__.py @@ -1,4 +1,4 @@ -from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchAndroid, info_main +from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchARM, info_main from os.path import join, exists from os import walk import glob @@ -19,8 +19,10 @@ def run_distribute(self): # self.name) src_path = join(self.bootstrap_dir, 'build') - # AND: Hardcoding armeabi - naughty! - arch = ArchAndroid(self.ctx) + arch = self.ctx.archs[0] + if len(self.ctx.archs) > 1: + raise ValueError('built for more than one arch, but bootstrap cannot handle that yet') + info('Bootstrap running with arch {}'.format(arch)) with current_directory(self.dist_dir): @@ -58,7 +60,7 @@ def run_distribute(self): shprint(sh.mkdir, '-p', join('private', 'include', 'python2.7')) # AND: Copylibs stuff should go here - shprint(sh.mv, join('libs', 'armeabi', 'libpymodules.so'), 'private/') + shprint(sh.mv, join('libs', arch.arch, 'libpymodules.so'), 'private/') shprint(sh.cp, join('python-install', 'include' , 'python2.7', 'pyconfig.h'), join('private', 'include', 'python2.7/')) info('Removing some unwanted files') diff --git a/pythonforandroid/bootstraps/pygame/build/jni/application/Android.mk b/pythonforandroid/bootstraps/pygame/build/jni/application/Android.mk index 00adb4d9af..75fb5d5ff5 100644 --- a/pythonforandroid/bootstraps/pygame/build/jni/application/Android.mk +++ b/pythonforandroid/bootstraps/pygame/build/jni/application/Android.mk @@ -18,7 +18,7 @@ LOCAL_CFLAGS := $(foreach D, $(APP_SUBDIRS), -I$(LOCAL_PATH)/$(D)) \ -I$(LOCAL_PATH)/../jpeg \ -I$(LOCAL_PATH)/../intl \ -I$(LOCAL_PATH)/.. \ - -I$(LOCAL_PATH)/../../../../other_builds/python2/armeabi/python2/python-install/include/python2.7 + -I$(LOCAL_PATH)/../../../../other_builds/python2/$(ARCH)/python2/python-install/include/python2.7 # -I$(LOCAL_PATH)/../../../../python-install/include/python2.7 # -I$(LOCAL_PATH)/../../../build/python-install/include/python2.7 @@ -39,7 +39,8 @@ LOCAL_STATIC_LIBRARIES := jpeg png LOCAL_LDLIBS := -lpython2.7 -lGLESv1_CM -ldl -llog -lz # AND: Another hardcoded path that should be templated -LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../other_builds/python2/armeabi/python2/python-install/lib $(APPLICATION_ADDITIONAL_LDFLAGS) +# AND: NOT TEMPALTED! We can use $ARCH +LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../other_builds/python2/$(ARCH)/python2/python-install/lib $(APPLICATION_ADDITIONAL_LDFLAGS) LIBS_WITH_LONG_SYMBOLS := $(strip $(shell \ for f in $(LOCAL_PATH)/../../libs/$ARCH/*.so ; do \ diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index 10c9e0f118..33871a2d10 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -1,4 +1,4 @@ -from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchAndroid, info_main +from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchARM, info_main from os.path import join, exists from os import walk import glob @@ -20,8 +20,10 @@ def run_distribute(self): with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) - # AND: Hardcoding armeabi - naughty! - arch = ArchAndroid(self.ctx) + arch = self.ctx.archs[0] + if len(self.ctx.archs) > 1: + raise ValueError('built for more than one arch, but bootstrap cannot handle that yet') + info('Bootstrap running with arch {}'.format(arch)) with current_directory(self.dist_dir): info('Copying python distribution') @@ -32,7 +34,6 @@ def run_distribute(self): shprint(sh.mkdir, 'assets') hostpython = sh.Command(self.ctx.hostpython) - # AND: This *doesn't* need to be in arm env? shprint(hostpython, '-OO', '-m', 'compileall', self.ctx.get_python_install_dir(), _tail=10, _filterout="^Listing", _critical=True) @@ -50,8 +51,8 @@ def run_distribute(self): shprint(sh.mkdir, '-p', join('private', 'include', 'python2.7')) # AND: Copylibs stuff should go here - if exists(join('libs', 'armeabi', 'libpymodules.so')): - shprint(sh.mv, join('libs', 'armeabi', 'libpymodules.so'), 'private/') + if exists(join('libs', arch.arch, 'libpymodules.so')): + shprint(sh.mv, join('libs', arch.arch, 'libpymodules.so'), 'private/') shprint(sh.cp, join('python-install', 'include' , 'python2.7', 'pyconfig.h'), join('private', 'include', 'python2.7/')) info('Removing some unwanted files') diff --git a/pythonforandroid/bootstraps/sdl2/build/jni/Application.mk b/pythonforandroid/bootstraps/sdl2/build/jni/Application.mk index 79b504d3f0..e79e378f94 100644 --- a/pythonforandroid/bootstraps/sdl2/build/jni/Application.mk +++ b/pythonforandroid/bootstraps/sdl2/build/jni/Application.mk @@ -4,4 +4,4 @@ # APP_STL := stlport_static # APP_ABI := armeabi armeabi-v7a x86 -APP_ABI := armeabi +APP_ABI := $(ARCH) diff --git a/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk b/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk index ec5eaa48ed..6cc877137b 100644 --- a/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk +++ b/pythonforandroid/bootstraps/sdl2/build/jni/src/Android.mk @@ -12,12 +12,12 @@ LOCAL_C_INCLUDES := $(LOCAL_PATH)/$(SDL_PATH)/include LOCAL_SRC_FILES := $(SDL_PATH)/src/main/android/SDL_android_main.c \ start.c -LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../other_builds/python2/armeabi/python2/python-install/include/python2.7 +LOCAL_CFLAGS += -I$(LOCAL_PATH)/../../../../other_builds/python2/$(ARCH)/python2/python-install/include/python2.7 LOCAL_SHARED_LIBRARIES := SDL2 LOCAL_LDLIBS := -lGLESv1_CM -lGLESv2 -llog -lpython2.7 -LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../other_builds/python2/armeabi/python2/python-install/lib $(APPLICATION_ADDITIONAL_LDFLAGS) +LOCAL_LDFLAGS += -L$(LOCAL_PATH)/../../../../other_builds/python2/$(ARCH)/python2/python-install/lib $(APPLICATION_ADDITIONAL_LDFLAGS) include $(BUILD_SHARED_LIBRARY) diff --git a/pythonforandroid/bootstraps/sdl2python3/__init__.py b/pythonforandroid/bootstraps/sdl2python3/__init__.py index 4218e827be..ae693e79eb 100644 --- a/pythonforandroid/bootstraps/sdl2python3/__init__.py +++ b/pythonforandroid/bootstraps/sdl2python3/__init__.py @@ -1,4 +1,4 @@ -from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchAndroid, info_main +from pythonforandroid.toolchain import Bootstrap, shprint, current_directory, info, warning, ArchARM, info_main from os.path import join, exists from os import walk import glob @@ -23,7 +23,7 @@ def run_distribute(self): fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) # AND: Hardcoding armeabi - naughty! - arch = ArchAndroid(self.ctx) + arch = ArchARM(self.ctx) with current_directory(self.dist_dir): info('Copying python distribution') diff --git a/pythonforandroid/recipes/android/__init__.py b/pythonforandroid/recipes/android/__init__.py index 2f669b5768..00da87a001 100644 --- a/pythonforandroid/recipes/android/__init__.py +++ b/pythonforandroid/recipes/android/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import CythonRecipe, shprint, ensure_dir, current_directory, ArchAndroid, IncludedFilesBehaviour +from pythonforandroid.toolchain import CythonRecipe, shprint, ensure_dir, current_directory, ArchARM, IncludedFilesBehaviour import sh from os.path import exists, join diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index 94082d9834..22f829b2a2 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchAndroid +from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchARM from os.path import exists, join, realpath from os import uname import glob @@ -12,8 +12,8 @@ class FreetypeRecipe(Recipe): depends = ['harfbuzz'] - def should_build(self): - if exists(join(self.get_build_dir('armeabi'), 'objs', '.libs', 'libfreetype.so')): + def should_build(self, arch): + if exists(join(self.get_build_dir(arch.arch), 'objs', '.libs', 'libfreetype.so')): return False return True diff --git a/pythonforandroid/recipes/harfbuzz/__init__.py b/pythonforandroid/recipes/harfbuzz/__init__.py index 74f3489a29..7c872ed2b9 100644 --- a/pythonforandroid/recipes/harfbuzz/__init__.py +++ b/pythonforandroid/recipes/harfbuzz/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchAndroid +from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchARM from os.path import exists, join, realpath from os import uname import glob @@ -10,8 +10,8 @@ class HarfbuzzRecipe(Recipe): version = '0.9.40' url = 'http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-{version}.tar.bz2' - def should_build(self): - if exists(join(self.get_build_dir('armeabi'), 'src', '.libs', 'libharfbuzz.so')): + def should_build(self, arch): + if exists(join(self.get_build_dir(arch.arch), 'src', '.libs', 'libharfbuzz.so')): return False return True diff --git a/pythonforandroid/recipes/hostpython2/__init__.py b/pythonforandroid/recipes/hostpython2/__init__.py index 3f8d974c14..6305e832c1 100644 --- a/pythonforandroid/recipes/hostpython2/__init__.py +++ b/pythonforandroid/recipes/hostpython2/__init__.py @@ -12,24 +12,27 @@ class Hostpython2Recipe(Recipe): conflicts = ['hostpython3'] - def prebuild_armeabi(self): + def get_build_container_dir(self, arch=None): + choices = self.check_recipe_choices() + dir_name = '-'.join([self.name] + choices) + return join(self.ctx.build_dir, 'other_builds', dir_name, 'desktop') + + def get_build_dir(self, arch=None): + return join(self.get_build_container_dir(), self.name) + + def prebuild_arch(self, arch): # Override hostpython Setup? shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'), - join(self.get_build_dir('armeabi'), 'Modules', 'Setup')) - - def build_armeabi(self): - # AND: Should use an i386 recipe system - warning('Running hostpython build. Arch is armeabi! ' - 'This is naughty, need to fix the Arch system!') + join(self.get_build_dir(), 'Modules', 'Setup')) - # AND: Fix armeabi again - with current_directory(self.get_build_dir('armeabi')): + def build_arch(self, arch): + with current_directory(self.get_build_dir()): if exists('hostpython'): info('hostpython already exists, skipping build') - self.ctx.hostpython = join(self.get_build_dir('armeabi'), + self.ctx.hostpython = join(self.get_build_dir(), 'hostpython') - self.ctx.hostpgen = join(self.get_build_dir('armeabi'), + self.ctx.hostpgen = join(self.get_build_dir(), 'hostpgen') return @@ -49,8 +52,8 @@ def build_armeabi(self): 'hostpython build! Exiting.') exit(1) - self.ctx.hostpython = join(self.get_build_dir('armeabi'), 'hostpython') - self.ctx.hostpgen = join(self.get_build_dir('armeabi'), 'hostpgen') + self.ctx.hostpython = join(self.get_build_dir(), 'hostpython') + self.ctx.hostpgen = join(self.get_build_dir(), 'hostpgen') recipe = Hostpython2Recipe() diff --git a/pythonforandroid/recipes/kivy/__init__.py b/pythonforandroid/recipes/kivy/__init__.py index f6add058c8..2430fa7aec 100644 --- a/pythonforandroid/recipes/kivy/__init__.py +++ b/pythonforandroid/recipes/kivy/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import CythonRecipe, shprint, current_directory, ArchAndroid +from pythonforandroid.toolchain import CythonRecipe, shprint, current_directory, ArchARM from os.path import exists, join import sh import glob diff --git a/pythonforandroid/recipes/kivysdl2python3/__init__.py b/pythonforandroid/recipes/kivysdl2python3/__init__.py index 63778560ea..22a2493294 100644 --- a/pythonforandroid/recipes/kivysdl2python3/__init__.py +++ b/pythonforandroid/recipes/kivysdl2python3/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import CythonRecipe, shprint, current_directory, ArchAndroid +from pythonforandroid.toolchain import CythonRecipe, shprint, current_directory, ArchARM from os.path import exists, join import sh import glob @@ -35,7 +35,7 @@ def get_recipe_env(self, arch): return env # def build_armeabi(self): - # env = ArchAndroid(self.ctx).get_env() + # env = ArchARM(self.ctx).get_env() # env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(self.ctx.libs_dir) # env['LDSHARED'] = env['LIBLINK'] diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 01cb0acba7..5c8ae5d0a9 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -20,10 +20,10 @@ def prebuild_arch(self, arch): print('numpy already patched, skipping') return - self.apply_patch('patches/fix-numpy.patch') - self.apply_patch('patches/prevent_libs_check.patch') - self.apply_patch('patches/ar.patch') - self.apply_patch('patches/lib.patch') + self.apply_patch('patches/fix-numpy.patch', arch.arch) + self.apply_patch('patches/prevent_libs_check.patch', arch.arch) + self.apply_patch('patches/ar.patch', arch.arch) + self.apply_patch('patches/lib.patch', arch.arch) # AND: Fix this warning! warning('Numpy is built assuming the archiver name is ' diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index 4a0afd0b88..77fc82cf56 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -8,8 +8,8 @@ class OpenSSLRecipe(Recipe): version = '1.0.2d' url = 'https://www.openssl.org/source/openssl-{version}.tar.gz' - def should_build(self): - return not exists(join(self.get_build_dir('armeabi'), 'libssl.a')) + def should_build(self, arch): + return not exists(join(self.get_build_dir(arch.arch), 'libssl.a')) def build_arch(self, arch): env = self.get_recipe_env(arch) diff --git a/pythonforandroid/recipes/pygame/__init__.py b/pythonforandroid/recipes/pygame/__init__.py index 6fcf33b500..231563ef80 100644 --- a/pythonforandroid/recipes/pygame/__init__.py +++ b/pythonforandroid/recipes/pygame/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import Recipe, shprint, ArchAndroid, current_directory, debug, info, ensure_dir +from pythonforandroid.toolchain import Recipe, shprint, ArchARM, current_directory, debug, info, ensure_dir from os.path import exists, join import sh import glob @@ -26,21 +26,20 @@ def get_recipe_env(self, arch): ensure_dir(liblink_path) return env - def prebuild_armeabi(self): - if exists(join(self.get_build_container_dir('armeabi'), '.patched')): + def prebuild_arch(self, arch): + if exists(join(self.get_build_container_dir(arch.arch), '.patched')): info('Pygame already patched, skipping.') return shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'), - join(self.get_build_dir('armeabi'), 'Setup')) - self.apply_patch(join('patches', 'fix-surface-access.patch')) - self.apply_patch(join('patches', 'fix-array-surface.patch')) - self.apply_patch(join('patches', 'fix-sdl-spam-log.patch')) - shprint(sh.touch, join(self.get_build_container_dir('armeabi'), '.patched')) + join(self.get_build_dir(arch.arch), 'Setup')) + self.apply_patch(join('patches', 'fix-surface-access.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-array-surface.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-sdl-spam-log.patch'), arch.arch) + shprint(sh.touch, join(self.get_build_container_dir(arch.arch), '.patched')) - def build_armeabi(self): + def build_arch(self, arch): # AND: I'm going to ignore any extra pythonrecipe or cythonrecipe behaviour for now - arch = ArchAndroid(self.ctx) env = self.get_recipe_env(arch) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format( @@ -57,7 +56,7 @@ def build_armeabi(self): env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink') - with current_directory(self.get_build_dir('armeabi')): + with current_directory(self.get_build_dir(arch.arch)): info('hostpython is ' + self.ctx.hostpython) hostpython = sh.Command(self.ctx.hostpython) shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, diff --git a/pythonforandroid/recipes/pyjnius/__init__.py b/pythonforandroid/recipes/pyjnius/__init__.py index 5fa643fd69..7ef1709c51 100644 --- a/pythonforandroid/recipes/pyjnius/__init__.py +++ b/pythonforandroid/recipes/pyjnius/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import CythonRecipe, shprint, ArchAndroid, current_directory, info +from pythonforandroid.toolchain import CythonRecipe, shprint, ArchARM, current_directory, info import sh import glob from os.path import join, exists @@ -18,7 +18,7 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): print('pyjniussdl2 already pathed, skipping') return - self.apply_patch('sdl2_jnienv_getter.patch') + self.apply_patch('sdl2_jnienv_getter.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) def postbuild_arch(self, arch): diff --git a/pythonforandroid/recipes/pyopenssl/__init__.py b/pythonforandroid/recipes/pyopenssl/__init__.py index fe3c995ac0..452ae85a46 100644 --- a/pythonforandroid/recipes/pyopenssl/__init__.py +++ b/pythonforandroid/recipes/pyopenssl/__init__.py @@ -22,7 +22,7 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): print('pyOpenSSL already patched, skipping') return - self.apply_patch('fix-dlfcn.patch') + self.apply_patch('fix-dlfcn.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) def get_recipe_env(self, arch): diff --git a/pythonforandroid/recipes/pysdl2/__init__.py b/pythonforandroid/recipes/pysdl2/__init__.py index 2f872f0cef..cd22b639ea 100644 --- a/pythonforandroid/recipes/pysdl2/__init__.py +++ b/pythonforandroid/recipes/pysdl2/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import PythonRecipe, shprint, current_directory, ArchAndroid +from pythonforandroid.toolchain import PythonRecipe, shprint, current_directory, ArchARM from os.path import exists, join import sh import glob diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index 41c12b2457..4e49a4fa5a 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchAndroid, info +from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchARM, info from os.path import exists, join, realpath from os import uname import glob @@ -15,44 +15,46 @@ class Python2Recipe(Recipe): conflicts = ['python3'] opt_depends = ['openssl'] - def prebuild_armeabi(self): - build_dir = self.get_build_container_dir('armeabi') + def prebuild_arch(self, arch): + build_dir = self.get_build_container_dir(arch.arch) if exists(join(build_dir, '.patched')): info('Python2 already patched, skipping.') return - self.apply_patch(join('patches', 'Python-{}-xcompile.patch'.format(self.version))) - self.apply_patch(join('patches', 'Python-{}-ctypes-disable-wchar.patch'.format(self.version))) - self.apply_patch(join('patches', 'disable-modules.patch')) - self.apply_patch(join('patches', 'fix-locale.patch')) - self.apply_patch(join('patches', 'fix-gethostbyaddr.patch')) - self.apply_patch(join('patches', 'fix-setup-flags.patch')) - self.apply_patch(join('patches', 'fix-filesystemdefaultencoding.patch')) - self.apply_patch(join('patches', 'fix-termios.patch')) - self.apply_patch(join('patches', 'custom-loader.patch')) - self.apply_patch(join('patches', 'verbose-compilation.patch')) - self.apply_patch(join('patches', 'fix-remove-corefoundation.patch')) - self.apply_patch(join('patches', 'fix-dynamic-lookup.patch')) - self.apply_patch(join('patches', 'fix-dlfcn.patch')) - self.apply_patch(join('patches', 'parsetuple.patch')) - # self.apply_patch(join('patches', 'ctypes-find-library.patch')) - self.apply_patch(join('patches', 'ctypes-find-library-updated.patch')) + self.apply_patch(join('patches', 'Python-{}-xcompile.patch'.format(self.version)), + arch.arch) + self.apply_patch(join('patches', 'Python-{}-ctypes-disable-wchar.patch'.format(self.version)), + arch.arch) + self.apply_patch(join('patches', 'disable-modules.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-locale.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-gethostbyaddr.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-setup-flags.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-filesystemdefaultencoding.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-termios.patch'), arch.arch) + self.apply_patch(join('patches', 'custom-loader.patch'), arch.arch) + self.apply_patch(join('patches', 'verbose-compilation.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-remove-corefoundation.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-dynamic-lookup.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-dlfcn.patch'), arch.arch) + self.apply_patch(join('patches', 'parsetuple.patch'), arch.arch) + # self.apply_patch(join('patches', 'ctypes-find-library.patch'), arch.arch) + self.apply_patch(join('patches', 'ctypes-find-library-updated.patch'), arch.arch) if uname()[0] == 'Linux': - self.apply_patch(join('patches', 'fix-configure-darwin.patch')) - self.apply_patch(join('patches', 'fix-distutils-darwin.patch')) + self.apply_patch(join('patches', 'fix-configure-darwin.patch'), arch.arch) + self.apply_patch(join('patches', 'fix-distutils-darwin.patch'), arch.arch) if self.ctx.android_api > 19: - self.apply_patch(join('patches', 'fix-ftime-removal.patch')) + self.apply_patch(join('patches', 'fix-ftime-removal.patch'), arch.arch) shprint(sh.touch, join(build_dir, '.patched')) - def build_armeabi(self): + def build_arch(self, arch): - if not exists(join(self.get_build_dir('armeabi'), 'libpython2.7.so')): - self.do_python_build() + if not exists(join(self.get_build_dir(arch.arch), 'libpython2.7.so')): + self.do_python_build(arch) if not exists(self.ctx.get_python_install_dir()): - shprint(sh.cp, '-a', join(self.get_build_dir('armeabi'), 'python-install'), + shprint(sh.cp, '-a', join(self.get_build_dir(arch.arch), 'python-install'), self.ctx.get_python_install_dir()) # This should be safe to run every time @@ -61,11 +63,11 @@ def build_armeabi(self): join(self.ctx.get_python_install_dir(), 'bin', 'python.host')) self.ctx.hostpython = join(self.ctx.get_python_install_dir(), 'bin', 'python.host') - if not exists(join(self.ctx.get_libs_dir('armeabi'), 'libpython2.7.so')): - shprint(sh.cp, join(self.get_build_dir('armeabi'), 'libpython2.7.so'), self.ctx.get_libs_dir('armeabi')) + if not exists(join(self.ctx.get_libs_dir(arch.arch), 'libpython2.7.so')): + shprint(sh.cp, join(self.get_build_dir(arch.arch), 'libpython2.7.so'), self.ctx.get_libs_dir(arch.arch)) - # # if exists(join(self.get_build_dir('armeabi'), 'libpython2.7.so')): + # # if exists(join(self.get_build_dir(arch.arch), 'libpython2.7.so')): # if exists(join(self.ctx.libs_dir, 'libpython2.7.so')): # info('libpython2.7.so already exists, skipping python build.') # if not exists(join(self.ctx.get_python_install_dir(), 'libpython2.7.so')): @@ -75,24 +77,24 @@ def build_armeabi(self): # return - def do_python_build(self): + def do_python_build(self, arch): if 'sqlite' in self.ctx.recipe_build_order: print('sqlite support not yet enabled in python recipe') exit(1) hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx) - shprint(sh.cp, self.ctx.hostpython, self.get_build_dir('armeabi')) - shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir('armeabi')) - hostpython = join(self.get_build_dir('armeabi'), 'hostpython') - hostpgen = join(self.get_build_dir('armeabi'), 'hostpython') + shprint(sh.cp, self.ctx.hostpython, self.get_build_dir(arch.arch)) + shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir(arch.arch)) + hostpython = join(self.get_build_dir(arch.arch), 'hostpython') + hostpgen = join(self.get_build_dir(arch.arch), 'hostpython') - with current_directory(self.get_build_dir('armeabi')): + with current_directory(self.get_build_dir(arch.arch)): hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx) shprint(sh.cp, join(hostpython_recipe.get_recipe_dir(), 'Setup'), 'Modules') - env = ArchAndroid(self.ctx).get_env() + env = arch.get_env() # AND: Should probably move these to get_recipe_env for # neatness, but the whole recipe needs tidying along these @@ -104,7 +106,7 @@ def do_python_build(self): # TODO need to add a should_build that checks if optional # dependencies have changed (possibly in a generic way) if 'openssl' in self.ctx.recipe_build_order: - openssl_build_dir = Recipe.get_recipe('openssl', self.ctx).get_build_dir('armeabi') + openssl_build_dir = Recipe.get_recipe('openssl', self.ctx).get_build_dir(arch.arch) env['CFLAGS'] = ' '.join([env['CFLAGS'], '-I{}'.format(join(openssl_build_dir, 'include'))]) env['LDFLAGS'] = ' '.join([env['LDFLAGS'], diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 0fd3edcb15..3bb7d84199 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -1,5 +1,5 @@ -from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchAndroid +from pythonforandroid.toolchain import Recipe, shprint, get_directory, current_directory, ArchARM from os.path import exists, join from os import uname import glob @@ -13,8 +13,8 @@ class Python3Recipe(Recipe): depends = ['hostpython3'] conflicts = ['python2'] - def prebuild_armeabi(self): - build_dir = self.get_build_container_dir('armeabi') + def prebuild_arch(self, arch): + build_dir = self.get_build_container_dir(arch.arch) if exists(join(build_dir, '.patched')): print('Python3 already patched, skipping.') return @@ -30,44 +30,45 @@ def prebuild_armeabi(self): # 'python-{version}-locale_and_android_misc.patch'.format(version=self.version))) - self.apply_patch(join('patches', 'python-{version}-android-libmpdec.patch'.format(version=self.version))) - self.apply_patch(join('patches', 'python-{version}-android-locale.patch'.format(version=self.version))) - self.apply_patch(join('patches', 'python-{version}-android-misc.patch'.format(version=self.version))) - # self.apply_patch(join('patches', 'python-{version}-android-missing-getdents64-definition.patch'.format(version=self.version))) - self.apply_patch(join('patches', 'python-{version}-cross-compile.patch'.format(version=self.version))) - self.apply_patch(join('patches', 'python-{version}-python-misc.patch'.format(version=self.version))) + self.apply_patch(join('patches', 'python-{version}-android-libmpdec.patch'.format(version=self.version)), + arch.arch) + self.apply_patch(join('patches', 'python-{version}-android-locale.patch'.format(version=self.version)), arch.arch) + self.apply_patch(join('patches', 'python-{version}-android-misc.patch'.format(version=self.version)), arch.arch) + # self.apply_patch(join('patches', 'python-{version}-android-missing-getdents64-definition.patch'.format(version=self.version)), arch.arch) + self.apply_patch(join('patches', 'python-{version}-cross-compile.patch'.format(version=self.version)), arch.arch) + self.apply_patch(join('patches', 'python-{version}-python-misc.patch'.format(version=self.version)), arch.arch) - self.apply_patch(join('patches', 'python-{version}-libpymodules_loader.patch'.format(version=self.version))) - self.apply_patch('log_failures.patch') + self.apply_patch(join('patches', 'python-{version}-libpymodules_loader.patch'.format(version=self.version)), arch.arch) + self.apply_patch('log_failures.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) - def build_armeabi(self): + def build_arch(self, arch): if 'sqlite' in self.ctx.recipe_build_order or 'openssl' in self.ctx.recipe_build_order: print('sqlite or openssl support not yet enabled in python recipe') exit(1) hostpython_recipe = Recipe.get_recipe('hostpython3', self.ctx) - shprint(sh.cp, self.ctx.hostpython, self.get_build_dir('armeabi')) - shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir('armeabi')) - hostpython = join(self.get_build_dir('armeabi'), 'hostpython') - hostpgen = join(self.get_build_dir('armeabi'), 'hostpython') + shprint(sh.cp, self.ctx.hostpython, self.get_build_dir(arch.arch)) + shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir(arch.arch)) + hostpython = join(self.get_build_dir(arch.arch), 'hostpython') + hostpgen = join(self.get_build_dir(arch.arch), 'hostpython') - if exists(join(self.get_build_dir('armeabi'), 'libpython3.4m.so')): + if exists(join(self.get_build_dir(arch.arch), 'libpython3.4m.so')): print('libpython3.4m.so already exists, skipping python build.') self.ctx.hostpython = join(self.ctx.build_dir, 'python-install', 'bin', 'python.host') return - with current_directory(self.get_build_dir('armeabi')): + with current_directory(self.get_build_dir(arch.arch)): hostpython_recipe = Recipe.get_recipe('hostpython3', self.ctx) # shprint(sh.cp, join(hostpython_recipe.get_recipe_dir(), 'Setup'), 'Modules') - env = ArchAndroid(self.ctx).get_env() + env = ArchARM(self.ctx).get_env() env["LDFLAGS"] = env["LDFLAGS"] + ' -llog' # AND: Should probably move these to get_recipe_env for diff --git a/pythonforandroid/recipes/sdl/__init__.py b/pythonforandroid/recipes/sdl/__init__.py index 1da3999d26..a8044f65e2 100644 --- a/pythonforandroid/recipes/sdl/__init__.py +++ b/pythonforandroid/recipes/sdl/__init__.py @@ -1,4 +1,4 @@ -from pythonforandroid.toolchain import NDKRecipe, shprint, ArchAndroid, current_directory, info +from pythonforandroid.toolchain import NDKRecipe, shprint, ArchARM, current_directory, info from os.path import exists, join import sh @@ -9,22 +9,22 @@ class LibSDLRecipe(NDKRecipe): depends = ['python2', 'pygame_bootstrap_components'] conflicts = ['sdl2'] - def build_armeabi(self): + def build_arch(self, arch): if exists(join(self.ctx.libs_dir, 'libsdl.so')): info('libsdl.so already exists, skipping sdl build.') return - env = ArchAndroid(self.ctx).get_env() + env = arch.get_env() with current_directory(self.get_jni_dir()): shprint(sh.ndk_build, 'V=1', _env=env, _tail=20, _critical=True) - libs_dir = join(self.ctx.bootstrap.build_dir, 'libs', 'armeabi') + libs_dir = join(self.ctx.bootstrap.build_dir, 'libs', arch.arch) import os contents = list(os.walk(libs_dir))[0][-1] for content in contents: - shprint(sh.cp, '-a', join(self.ctx.bootstrap.build_dir, 'libs', 'armeabi', content), + shprint(sh.cp, '-a', join(self.ctx.bootstrap.build_dir, 'libs', arch.arch, content), self.ctx.libs_dir) diff --git a/pythonforandroid/recipes/sdl2/__init__.py b/pythonforandroid/recipes/sdl2/__init__.py index aeb436915b..9edec09842 100644 --- a/pythonforandroid/recipes/sdl2/__init__.py +++ b/pythonforandroid/recipes/sdl2/__init__.py @@ -18,7 +18,7 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): info('SDL2 already patched, skipping') return - self.apply_patch('add_nativeSetEnv.patch') + self.apply_patch('add_nativeSetEnv.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) def build_arch(self, arch): diff --git a/pythonforandroid/recipes/sdl2_image/__init__.py b/pythonforandroid/recipes/sdl2_image/__init__.py index e4444ba0a3..8265d4ec88 100644 --- a/pythonforandroid/recipes/sdl2_image/__init__.py +++ b/pythonforandroid/recipes/sdl2_image/__init__.py @@ -13,7 +13,9 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): info('SDL2_image already patched, skipping') return - self.apply_patch('disable_webp.patch') + self.apply_patch('disable_webp.patch', arch.arch) + if arch.arch == 'x86': + self.apply_patch('disable_jpg.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) recipe = LibSDL2Image() diff --git a/pythonforandroid/recipes/sdl2_image/disable_jpg.patch b/pythonforandroid/recipes/sdl2_image/disable_jpg.patch new file mode 100644 index 0000000000..2b5fc38280 --- /dev/null +++ b/pythonforandroid/recipes/sdl2_image/disable_jpg.patch @@ -0,0 +1,13 @@ +diff --git a/Android.mk b/Android.mk +index 31e2118..8cd2cb9 100644 +--- a/Android.mk ++++ b/Android.mk +@@ -6,7 +6,7 @@ LOCAL_MODULE := SDL2_image + + # Enable this if you want to support loading JPEG images + # The library path should be a relative path to this directory. +-SUPPORT_JPG := true ++SUPPORT_JPG := false + JPG_LIBRARY_PATH := external/jpeg-9 + + # Enable this if you want to support loading PNG images diff --git a/pythonforandroid/recipes/sdl2_mixer/__init__.py b/pythonforandroid/recipes/sdl2_mixer/__init__.py index d832fd8437..8788330363 100644 --- a/pythonforandroid/recipes/sdl2_mixer/__init__.py +++ b/pythonforandroid/recipes/sdl2_mixer/__init__.py @@ -14,7 +14,8 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): info('SDL2_mixer already patched, skipping') return - self.apply_patch('disable_modplug_mikmod_smpeg.patch') + self.apply_patch('disable_modplug_mikmod_smpeg.patch', + arch.arch) shprint(sh.touch, join(build_dir, '.patched')) recipe = LibSDL2Mixer() diff --git a/pythonforandroid/recipes/sdl2python3/__init__.py b/pythonforandroid/recipes/sdl2python3/__init__.py index 7e3b6f9ccf..8c51684c9d 100644 --- a/pythonforandroid/recipes/sdl2python3/__init__.py +++ b/pythonforandroid/recipes/sdl2python3/__init__.py @@ -17,7 +17,7 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): print('SDL2 already patched, skipping') return - self.apply_patch('add_nativeSetEnv.patch') + self.apply_patch('add_nativeSetEnv.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) def build_arch(self, arch): diff --git a/pythonforandroid/recipes/vispy/__init__.py b/pythonforandroid/recipes/vispy/__init__.py index cf1b814ded..130a500009 100644 --- a/pythonforandroid/recipes/vispy/__init__.py +++ b/pythonforandroid/recipes/vispy/__init__.py @@ -22,13 +22,13 @@ def prebuild_arch(self, arch): if exists(join(build_dir, '.patched')): print('vispy already patched, skipping') return - self.apply_patch('disable_freetype.patch') - self.apply_patch('disable_font_triage.patch') - self.apply_patch('use_es2.patch') - self.apply_patch('remove_ati_check.patch') + self.apply_patch('disable_freetype.patch', arch.arch) + self.apply_patch('disable_font_triage.patch', arch.arch) + self.apply_patch('use_es2.patch', arch.arch) + self.apply_patch('remove_ati_check.patch', arch.arch) - self.apply_patch('make_shader_es2_compliant.patch') - self.apply_patch('detect_finger_events.patch') + self.apply_patch('make_shader_es2_compliant.patch', arch.arch) + self.apply_patch('detect_finger_events.patch', arch.arch) shprint(sh.touch, join(build_dir, '.patched')) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 3dbb16f9d7..1afe76bd5e 100755 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -11,8 +11,8 @@ import sys from sys import stdout, stderr, platform from os.path import (join, dirname, realpath, exists, isdir, basename, - expanduser, splitext) -from os import listdir, unlink, makedirs, environ, chdir, getcwd, walk, uname + expanduser, splitext, split) +from os import listdir, unlink, makedirs, environ, chdir, getcwd, uname import os import zipfile import tarfile @@ -56,7 +56,7 @@ def __init__(self): def __getattr__(self, key): return self._dict[key] - + Null_Style = Null_Fore = colorama_shim() if stdout.isatty(): @@ -72,6 +72,8 @@ def __getattr__(self, key): else: Err_Style = Null_Style Err_Fore = Null_Fore +Fore = Colo_Fore +Style = Colo_Style user_dir = dirname(realpath(os.path.curdir)) toolchain_dir = dirname(__file__) @@ -139,21 +141,28 @@ def pretty_log_dists(dists, log_func=info): for dist in dists: infos.append('{Fore.GREEN}{Style.BRIGHT}{name}{Style.RESET_ALL}: ' 'includes recipes ({Fore.GREEN}{recipes}' - '{Style.RESET_ALL})'.format( + '{Style.RESET_ALL}), built for archs ({Fore.BLUE}' + '{archs}{Style.RESET_ALL})'.format( name=dist.name, recipes=', '.join(dist.recipes), + archs=', '.join(dist.archs) if dist.archs else 'UNKNOWN', Fore=Err_Fore, Style=Err_Style)) for line in infos: log_func('\t' + line) + def shorten_string(string, max_width): ''' make limited length string in form: "the string is very lo...(and 15 more)" - ''' + ''' string_len = len(string) - if string_len <= max_width: return string - visible = max_width - 16 - int(log10(string_len)) #expected suffix len "...(and XXXXX more)" - return ''.join((string[:visible], '...(and ', str(string_len - visible), ' more)')) + if string_len <= max_width: + return string + visible = max_width - 16 - int(log10(string_len)) + # expected suffix len "...(and XXXXX more)" + return ''.join((string[:visible], '...(and ', str(string_len - visible), + ' more)')) + def shprint(command, *args, **kwargs): '''Runs the command (which should be an sh.Command instance), while @@ -286,6 +295,7 @@ def require_prebuilt_dist(func): @wraps(func) def wrapper_func(self, args): ctx = self.ctx + ctx.set_archs(self._archs) ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, @@ -348,17 +358,19 @@ def current_directory(new_dir): Err_Fore.RESET))) chdir(cur_dir) + @contextlib.contextmanager def temp_directory(): temp_dir = mkdtemp() try: - logger.debug(''.join((Err_Fore.CYAN, ' + temp directory used ', temp_dir, - Err_Fore.RESET))) + logger.debug(''.join((Err_Fore.CYAN, ' + temp directory used ', + temp_dir, Err_Fore.RESET))) yield temp_dir finally: shutil.rmtree(temp_dir) - logger.debug(''.join((Err_Fore.CYAN, ' - temp directory deleted ', temp_dir, - Err_Fore.RESET))) + logger.debug(''.join((Err_Fore.CYAN, ' - temp directory deleted ', + temp_dir, Err_Fore.RESET))) + def cache_execution(f): def _cache_execution(self, *args, **kwargs): @@ -441,6 +453,13 @@ def sync(self): class Arch(object): + + toolchain_prefix = None + '''The prefix for the toolchain dir in the NDK.''' + + command_prefix = None + '''The prefix for NDK commands such as gcc.''' + def __init__(self, ctx): super(Arch, self).__init__() self.ctx = ctx @@ -483,6 +502,9 @@ def get_env(self): env['TOOLCHAIN_PREFIX'] = toolchain_prefix env['TOOLCHAIN_VERSION'] = toolchain_version + if toolchain_prefix == 'x86': + toolchain_prefix = 'i686-linux-android' + print('path is', environ['PATH']) cc = find_executable('{toolchain_prefix}-gcc'.format( toolchain_prefix=toolchain_prefix), path=environ['PATH']) if cc is None: @@ -509,57 +531,61 @@ def get_env(self): hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx) # AND: This hardcodes python version 2.7, needs fixing - # AND: This also hardcodes armeabi, which isn't even correct, - # don't forget to fix! env['BUILDLIB_PATH'] = join( - hostpython_recipe.get_build_dir('armeabi'), + hostpython_recipe.get_build_dir(self.arch), 'build', 'lib.linux-{}-2.7'.format(uname()[-1])) env['PATH'] = environ['PATH'] - # AND: This stuff is set elsewhere in distribute.sh. Does that matter? env['ARCH'] = self.arch - # env['LIBLINK_PATH'] = join( - # self.ctx.build_dir, 'other_builds', 'objects') - # ensure_dir(env['LIBLINK_PATH']) # AND: This should be elsewhere - return env -class ArchAndroid(Arch): +class ArchARM(Arch): arch = "armeabi" + toolchain_prefix = 'arm-linux-androideabi' + command_prefix = 'arm-linux-androideabi' + platform_dir = 'arch-arm' -# class ArchSimulator(Arch): -# sdk = "iphonesimulator" -# arch = "i386" -# triple = "i386-apple-darwin11" -# version_min = "-miphoneos-version-min=6.0.0" -# sysroot = sh.xcrun("--sdk", "iphonesimulator", "--show-sdk-path").strip() +class ArchARMv7_a(ArchARM): + arch = 'armeabi-v7a' -# class Arch64Simulator(Arch): -# sdk = "iphonesimulator" -# arch = "x86_64" -# triple = "x86_64-apple-darwin13" -# version_min = "-miphoneos-version-min=7.0" -# sysroot = sh.xcrun("--sdk", "iphonesimulator", "--show-sdk-path").strip() + def get_env(self): + env = super(ArchARMv7_a, self).get_env() + env['CFLAGS'] = (env['CFLAGS'] + + ' -march=armv7-a -mfloat-abi=softfp -mfpu=vfp -mthumb') + env['CXXFLAGS'] = env['CFLAGS'] + return env -# class ArchIOS(Arch): -# sdk = "iphoneos" -# arch = "armv7" -# triple = "arm-apple-darwin11" -# version_min = "-miphoneos-version-min=6.0.0" -# sysroot = sh.xcrun("--sdk", "iphoneos", "--show-sdk-path").strip() +class Archx86(Arch): + arch = 'x86' + toolchain_prefix = 'x86' + command_prefix = 'i686-linux-android' + platform_dir = 'arch-x86' + def get_env(self): + env = super(Archx86, self).get_env() + env['CFLAGS'] = (env['CFLAGS'] + + ' -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32') + env['CXXFLAGS'] = env['CFLAGS'] + return env -# class Arch64IOS(Arch): -# sdk = "iphoneos" -# arch = "arm64" -# triple = "aarch64-apple-darwin13" -# version_min = "-miphoneos-version-min=7.0" -# sysroot = sh.xcrun("--sdk", "iphoneos", "--show-sdk-path").strip() + +class Archx86_64(Arch): + arch = 'x86_64' + toolchain_prefix = 'x86' + command_prefix = 'x86_64-linux-android' + platform_dir = 'arch-x86' + + def get_env(self): + env = super(Archx86_64, self).get_env() + env['CFLAGS'] = (env['CFLAGS'] + + ' -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel') + env['CXXFLAGS'] = env['CFLAGS'] + return env class Graph(object): @@ -869,8 +895,6 @@ def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, 'it with the SDK android tool.').format(android_api)) warning('Exiting.') exit(1) - # AND: If the android api target doesn't exist, we should - # offer to install it here # Find the Android NDK # Could also use ANDROID_NDK, but doesn't look like many tools use this @@ -944,15 +968,6 @@ def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, warning('Android NDK version could not be found, exiting.') self.ndk_ver = ndk_ver - self.ndk_platform = join( - self.ndk_dir, - 'platforms', - 'android-{}'.format(self.android_api), - 'arch-arm') - if not exists(self.ndk_platform): - warning('ndk_platform doesn\'t exist') - ok = False - virtualenv = None if virtualenv is None: virtualenv = sh.which('virtualenv2') @@ -980,33 +995,31 @@ def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, ok = False warning("Missing requirement: cython is not installed") - # Modify the path so that sh finds modules appropriately + # AND: need to change if supporting multiple archs at once + arch = self.archs[0] + platform_dir = arch.platform_dir + toolchain_prefix = arch.toolchain_prefix + command_prefix = arch.command_prefix + self.ndk_platform = join( + self.ndk_dir, + 'platforms', + 'android-{}'.format(self.android_api), + platform_dir) + if not exists(self.ndk_platform): + warning('ndk_platform doesn\'t exist') + ok = False + py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' - if self.ndk_ver == 'r5b': - toolchain_prefix = 'arm-eabi' - elif self.ndk_ver[:2] in ('r7', 'r8', 'r9'): - toolchain_prefix = 'arm-linux-androideabi' - elif self.ndk_ver[:3] == 'r10': - toolchain_prefix = 'arm-linux-androideabi' - else: - warning('Error: NDK not supported by these tools?') - exit(1) toolchain_versions = [] toolchain_path = join(self.ndk_dir, 'toolchains') if os.path.isdir(toolchain_path): - toolchain_contents = os.listdir(toolchain_path) - for toolchain_content in toolchain_contents: - if toolchain_content.startswith(toolchain_prefix) and \ - os.path.isdir( - os.path.join(toolchain_path, toolchain_content)): - toolchain_version = toolchain_content[ - len(toolchain_prefix)+1:] - debug('Found toolchain version: {}'.format( - toolchain_version)) - toolchain_versions.append(toolchain_version) + toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, + toolchain_prefix)) + toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] + for path in toolchain_contents] else: warning('Could not find toolchain subdirectory!') ok = False @@ -1031,6 +1044,7 @@ def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, self.toolchain_prefix = toolchain_prefix self.toolchain_version = toolchain_version + # Modify the path so that sh finds modules appropriately environ['PATH'] = ( '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' @@ -1042,8 +1056,6 @@ def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, toolchain_version=toolchain_version, py_platform=py_platform, path=environ.get('PATH')) - # AND: Are these necessary? Where to check for and and ndk-build? - # check the basic tools for executable in ("pkg-config", "autoconf", "automake", "libtoolize", "tar", "bzip2", "unzip", "make", "gcc", "g++"): if not sh.which(executable): @@ -1070,9 +1082,11 @@ def __init__(self): # root of the toolchain self.setup_dirs() - # AND: Currently only the Android architecture is supported + # this list should contain all Archs, it is pruned later self.archs = ( - ArchAndroid(self), + ArchARM(self), + ArchARMv7_a(self), + Archx86(self) ) ensure_dir(join(self.build_dir, 'bootstrap_builds')) @@ -1087,6 +1101,20 @@ def __init__(self): # set the state self.state = JsonStore(join(self.dist_dir, "state.db")) + def set_archs(self, arch_names): + all_archs = self.archs + new_archs = set() + for name in arch_names: + matching = [arch for arch in all_archs if arch.arch == name] + for match in matching: + new_archs.add(match) + self.archs = list(new_archs) + if not self.archs: + warning('Asked to compile for no Archs, so failing.') + exit(1) + info('Will compile for the following archs: {}'.format( + ', '.join([arch.arch for arch in self.archs]))) + def prepare_bootstrap(self, bs): bs.ctx = self self.bootstrap = bs @@ -1111,7 +1139,6 @@ def get_site_packages_dir(self, arch=None): def get_libs_dir(self, arch): '''The libs dir for a given arch.''' ensure_dir(join(self.libs_dir, arch)) - # AND: See warning: return join(self.libs_dir, arch) @@ -1130,6 +1157,9 @@ class Distribution(object): url = None dist_dir = None # Where the dist dir ultimately is. Should not be None. + archs = [] + '''The arch targets that the dist is built for.''' + recipes = [] description = '' # A long description @@ -1287,13 +1317,12 @@ def get_distributions(cls, ctx, extra_dist_dirs=[]): with open(join(folder, 'dist_info.json')) as fileh: dist_info = json.load(fileh) dist = cls(ctx) - dist.name = folder.split('/')[-1] # AND: also equal to - # # dist_info['dist_name'] - # # ... Which one should we - # # use? + dist.name = folder.split('/')[-1] dist.dist_dir = folder dist.needs_build = False dist.recipes = dist_info['recipes'] + if 'archs' in dist_info: + dist.archs = dist_info['archs'] dists.append(dist) return dists @@ -1305,6 +1334,7 @@ def save_info(self): info('Saving distribution info') with open('dist_info.json', 'w') as fileh: json.dump({'dist_name': self.name, + 'archs': [arch.arch for arch in self.ctx.archs], 'recipes': self.ctx.recipe_build_order}, fileh) @@ -1399,14 +1429,10 @@ def run_distribute(self): with open('dist_info.json', 'w') as fileh: json.dump({'dist_name': self.ctx.dist_name, 'bootstrap': self.ctx.bootstrap.name, + 'archs': [arch.arch for arch in self.ctx.archs], 'recipes': self.ctx.recipe_build_order}, fileh) - # AND: This method must be replaced by manual dir setting, in - # order to allow for user dirs - # def get_bootstrap_dir(self): - # return(dirname(__file__)) - @classmethod def list_bootstraps(cls): '''Find all the available bootstraps and return them.''' @@ -1680,7 +1706,7 @@ def extract_source(self, source, cwd): # print("Unrecognized extension for {}".format(filename)) # raise Exception() - def apply_patch(self, filename, arch='armeabi'): + def apply_patch(self, filename, arch): """ Apply a patch from the current recipe directory into the current build directory. @@ -1794,12 +1820,6 @@ def get_recipe_dir(self): # Public Recipe API to be subclassed if needed - def ensure_build_container_dir(self): - info_main('Preparing build dir for {}'.format(self.name)) - - build_dir = self.get_build_container_dir('armeabi') - ensure_dir(build_dir) - def download_if_necessary(self): info_main('Downloading {}'.format(self.name)) user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower())) @@ -1874,7 +1894,7 @@ def unpack(self, arch): ensure_dir(build_dir) # shprint(sh.ln, '-s', user_dir, # join(build_dir, get_directory(self.versioned_url))) - shprint(sh.git, 'clone', user_dir, self.get_build_dir('armeabi')) + shprint(sh.git, 'clone', user_dir, self.get_build_dir(arch)) return if self.url is None: @@ -1884,7 +1904,6 @@ def unpack(self, arch): filename = shprint( sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') - # AND: TODO: Use tito's better unpacking method with current_directory(build_dir): directory_name = self.get_build_dir(arch) @@ -1944,46 +1963,6 @@ def get_recipe_env(self, arch=None): arch = self.filtered_archs[0] return arch.get_env() - # @property - # def archive_root(self): - # key = "{}.archive_root".format(self.name) - # value = self.ctx.state.get(key) - # if not key: - # value = self.get_archive_rootdir(self.archive_fn) - # self.ctx.state[key] = value - # return value - - # def execute(self): - # if self.custom_dir: - # self.ctx.state.remove_all(self.name) - # self.download() - # self.extract() - # self.build_all() - - # AND: Will need to change how this works - # @property - # def custom_dir(self): - # """Check if there is a variable name to specify a custom version / - # directory to use instead of the current url. - # """ - # d = environ.get("P4A_{}_DIR".format(self.name.lower())) - # if not d: - # return - # if not exists(d): - # return - # return d - - # def prebuild(self): - # self.prebuild_arch(self.ctx.archs[0]) # AND: Need to change - # # this to support - # # multiple archs - - # def build(self): - # self.build_arch(self.ctx.archs[0]) # Same here! - - # def postbuild(self): - # self.postbuild_arch(self.ctx.archs[0]) - def prebuild_arch(self, arch): '''Run any pre-build tasks for the Recipe. By default, this checks if any prebuild_archname methods exist for the archname of the current @@ -1994,12 +1973,11 @@ def prebuild_arch(self, arch): else: info('{} has no {}, skipping'.format(self.name, prebuild)) - def should_build(self): + def should_build(self, arch): '''Should perform any necessary test and return True only if it needs building again. ''' - # AND: This should take arch as an argument! return True def build_arch(self, arch): @@ -2121,20 +2099,6 @@ def get_build_dir(self, arch): def get_jni_dir(self): return join(self.ctx.bootstrap.build_dir, 'jni') - # def download_if_necessary(self): - # info_main('Downloading {}'.format(self.name)) - # info('{} is an NDK recipe, it is alread included in the ' - # 'bootstrap (for now), so skipping'.format(self.name)) - # # Do nothing; in the future an NDKRecipe can copy its - # # contents to the bootstrap build dir, but for now each - # # bootstrap already includes available recipes (as was - # # already the case in p4a) - - # def prepare_build_dir(self, arch): - # info_main('Unpacking {} for {}'.format(self.name, arch)) - # info('{} is included in the bootstrap, unpacking currently ' - # 'unnecessary, so skipping'.format(self.name)) - class PythonRecipe(Recipe): site_packages_name = None @@ -2159,14 +2123,11 @@ class PythonRecipe(Recipe): def hostpython_location(self): if not self.call_hostpython_via_targetpython: return join( - Recipe.get_recipe('hostpython2', self.ctx).get_build_dir( - 'armeabi'), 'hostpython') + Recipe.get_recipe('hostpython2', self.ctx).get_build_dir(), + 'hostpython') return self.ctx.hostpython - def should_build(self): - # AND: This should be different for each arch and use some - # kind of data store to know what has been built in a given - # python env + def should_build(self, arch): print('name is', self.site_packages_name, type(self)) name = self.site_packages_name if name is None: @@ -2181,16 +2142,12 @@ def build_arch(self, arch): '''Install the Python module by calling setup.py install with the target Python dir.''' super(PythonRecipe, self).build_arch(arch) - self.install_python_package() - # @cache_execution - # def install(self): - # self.install_python_package() - # self.reduce_python_package() + self.install_python_package(arch) - def install_python_package(self, name=None, env=None, is_dir=True): + def install_python_package(self, arch, name=None, env=None, is_dir=True): '''Automate the installation of a Python package (or a cython package where the cython components are pre-built).''' - arch = self.filtered_archs[0] + # arch = self.filtered_archs[0] # old kivy-ios way if name is None: name = self.name if env is None: @@ -2225,14 +2182,9 @@ def build_arch(self, arch): '''Build any cython components, then install the Python module by calling setup.py install with the target Python dir. ''' - Recipe.build_arch(self, arch) # AND: Having to directly call the - # # method like this is nasty...could - # # use tito's method of having an - # # install method that always runs - # # after everything else but isn't - # # used by a normal recipe. + Recipe.build_arch(self, arch) self.build_compiled_components(arch) - self.install_python_package() + self.install_python_package(arch) def build_compiled_components(self, arch): info('Building compiled components in {}'.format(self.name)) @@ -2254,17 +2206,11 @@ def build_arch(self, arch): '''Build any cython components, then install the Python module by calling setup.py install with the target Python dir. ''' - Recipe.build_arch(self, arch) # AND: Having to directly call the - # # method like this is nasty...could - # # use tito's method of having an - # # install method that always runs - # # after everything else but isn't - # # used by a normal recipe. + Recipe.build_arch(self, arch) self.build_cython_components(arch) - self.install_python_package() + self.install_python_package(arch) def build_cython_components(self, arch): - # AND: Should we use tito's cythonize methods? How do they work? info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): @@ -2278,7 +2224,7 @@ def build_cython_components(self, arch): info('{} first build failed (as expected)'.format(self.name)) info('Running cython where appropriate') - shprint(sh.find, self.get_build_dir('armeabi'), '-iname', '*.pyx', + shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx', '-exec', self.ctx.cython, '{}', ';', _env=env) info('ran cython') @@ -2307,15 +2253,6 @@ def build_cython_components(self, arch): # for filename in fnmatch.filter(filenames, "*.pyx"): # self.cythonize_file(join(root, filename)) - # def biglink(self): - # dirs = [] - # for root, dirnames, filenames in walk(self.build_dir): - # if fnmatch.filter(filenames, "*.so.libs"): - # dirs.append(root) - # cmd = sh.Command(join(self.ctx.root_dir, "tools", "biglink")) - # shprint(cmd, join(self.build_dir, "lib{}.a".format(self.name)), - # *dirs) - def get_recipe_env(self, arch): env = super(CythonRecipe, self).get_recipe_env(arch) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( @@ -2369,14 +2306,14 @@ def build_recipes(build_order, python_modules, ctx): info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) - if recipe.should_build(): + if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) # 4) biglink everything - # AND: Should make this optional (could use + # AND: Should make this optional info_main('# Biglinking object files') biglink(ctx, arch) @@ -2444,12 +2381,9 @@ def biglink(ctx, arch): files.append(obj_dir) shprint(sh.cp, '-r', *files) - # AND: Shouldn't hardcode ArchAndroid! In reality need separate - # build dirs for each arch - arch = ArchAndroid(ctx) - env = ArchAndroid(ctx).get_env() + env = arch.get_env() env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( - join(ctx.bootstrap.build_dir, 'obj', 'local', 'armeabi')) + join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)) if not len(glob.glob(join(obj_dir, '*'))): info('There seem to be no libraries to biglink, skipping.') @@ -2461,7 +2395,7 @@ def biglink(ctx, arch): join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), extra_link_dirs=[join(ctx.bootstrap.build_dir, - 'obj', 'local', 'armeabi')], + 'obj', 'local', arch.arch)], env=env) @@ -2505,21 +2439,11 @@ def biglink_function(soname, objs_paths, extra_link_dirs=[], env=None): if link not in unique_args: unique_args.append(link) - # print('Biglink create %s library' % soname) - # print('Biglink arguments:') - # for arg in unique_args: - # print(' %s' % arg) - cc_name = env['CC'] cc = sh.Command(cc_name.split()[0]) cc = cc.bake(*cc_name.split()[1:]) shprint(cc, '-shared', '-O3', '-o', soname, *unique_args, _env=env) - # args = os.environ['CC'].split() + \ - # ['-shared', '-O3', '-o', soname] + \ - # unique_args - - # sys.exit(subprocess.call(args)) def ensure_dir(filename): @@ -2754,6 +2678,14 @@ def __init__(self): help=('The version of the Android NDK. This is optional, ' 'we try to work it out automatically from the ndk_dir.')) + + # AND: This option doesn't really fit in the other categories, the + # arg structure needs a rethink + parser.add_argument( + '--arch', + help='The archs to build for, separated by commas.', + default='armeabi') + # Options for specifying the Distribution parser.add_argument( '--dist_name', @@ -2791,6 +2723,7 @@ def __init__(self): description=('Whether the dist recipes must perfectly match ' 'those requested')) + self._read_configuration() args, unknown = parser.parse_known_args(sys.argv[1:]) @@ -2803,8 +2736,8 @@ def __init__(self): self.android_api = args.android_api self.ndk_version = args.ndk_version - # import ipdb - # ipdb.set_trace() + self._archs = split_argument_list(args.arch) + # AND: Fail nicely if the args aren't handled yet if args.extra_dist_dirs: warning('Received --extra_dist_dirs but this arg currently is not ' @@ -2825,21 +2758,6 @@ def __init__(self): exit(1) getattr(self, args.command)(unknown) - # def build(self): - # parser = argparse.ArgumentParser( - # description="Build the toolchain") - # parser.add_argument("recipe", nargs="+", help="Recipe to compile") - # parser.add_argument("--arch", - # help="Restrict compilation to this arch") - # args = parser.parse_args(sys.argv[2:]) - - # ctx = Context() - # # if args.arch: - # # archs = args.arch.split() - # # ctx.archs = [arch for arch in ctx.archs if arch.arch in archs] - # # print("Architectures restricted to: {}".format(archs)) - # build_recipes(args.recipe, ctx) - def _read_configuration(self): # search for a .p4a configuration file in the current directory if not exists(".p4a"): @@ -2983,18 +2901,6 @@ def clean_download_cache(self, args): if exists(ctx.packages_path): shutil.rmtree(ctx.packages_path) - # def status(self, args): - # parser = argparse.ArgumentParser( - # description="Give a status of the build") - # args = parser.parse_args(args) - # ctx = Context() - # # AND: TODO - - # print('This isn\'t implemented yet, but should list all ' - # 'currently existing distributions, the modules they ' - # 'include, and all the build caches.') - # exit(1) - @require_prebuilt_dist def export_dist(self, args): '''Copies a created dist to an output dir. @@ -3119,6 +3025,13 @@ def print_context_info(self, args): 'ndk_platform', 'ndk_ver', 'android_api'): print('{} is {}'.format(attribute, getattr(ctx, attribute))) + def archs(self, args): + '''List the target architectures available to be built for.''' + print('{Style.BRIGHT}Available target architectures are:' + '{Style.RESET_ALL}'.format(Style=Out_Style)) + for arch in self.ctx.archs: + print(' {}'.format(arch.arch)) + def dists(self, args): '''The same as :meth:`distributions`.''' self.distributions(args)