From e00bfd0262bd0094c88f299b49756c40f7ee9b9e Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Thu, 15 Mar 2018 19:03:39 +0100 Subject: [PATCH 01/72] Started porting CPython 3.5.3 --- Makefile | 10 +- python_build/Makefile | 10 +- python_config/pyconfig.h | 543 ++++++++++++++++++++++++++++----------- 3 files changed, 404 insertions(+), 159 deletions(-) diff --git a/Makefile b/Makefile index b228a2e..d0f1b55 100644 --- a/Makefile +++ b/Makefile @@ -33,14 +33,14 @@ TARGET := $(notdir $(CURDIR)) BUILD := build PY_BUILD := python_build DIST_DIR := $(BUILD)/$(TARGET) -LIBDIR := $(DIST_DIR)/lib/python2.7 +LIBDIR := $(DIST_DIR)/lib/python3.5 SOURCES := source DATA := data INCLUDES := include EXEFS_SRC := exefs_src APP_TITLEID := Pynx APP_AUTHOR := nx-python Authors, Python Software Foundation -APP_VERSION := 0.1.0-alpha +APP_VERSION := 0.2.0-alpha #--------------------------------------------------------------------------------- # options for code generation @@ -57,13 +57,13 @@ CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions -std=gnu++11 ASFLAGS := -g $(ARCH) LDFLAGS = -specs=$(DEVKITPRO)/libnx/switch.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map) -LIBS := -lpython2.7 -lm -lnx +LIBS := -lpython3.5 -lm -lnx #--------------------------------------------------------------------------------- # list of directories containing libraries, this must be the top level containing # include and lib #--------------------------------------------------------------------------------- -NXPY := $(CURDIR)/python_build/nxpy2.7.12 +NXPY := $(CURDIR)/python_build/nxpy3.5.3 LIBDIRS := $(PORTLIBS) $(LIBNX) $(NXPY) @@ -151,7 +151,7 @@ cpython: dist: $(BUILD) mkdir -p $(LIBDIR) - unzip $(PY_BUILD)/nxpy2.7.12/python.zip -d $(LIBDIR) + unzip $(PY_BUILD)/nxpy3.5.3/python.zip -d $(LIBDIR) cp $(OUTPUT).{nro,nacp} $(DIST_DIR)/ cp examples/hello.py $(DIST_DIR)/main.py cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) diff --git a/python_build/Makefile b/python_build/Makefile index 87a5892..dea69bd 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -39,7 +39,7 @@ CFLAGS += $(INCLUDE) -DSWITCH ifndef PYVERS - PYVERS := 2.7.12 + PYVERS := 3.5.3 endif ANAME := libpython$(shell echo $(PYVERS) | sed 's/\([0-9]*\.\([0-9]*\)\).*/\1/').a @@ -78,10 +78,10 @@ soospatchPY: compilePY cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/access(path, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*i = (int)umask(i);.*\)/return NULL; \1/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old cat $(PYDIR)/Modules/socketmodule.c_old | sed 's/send(s->sock_fd, buf, len, flags);/send(s->sock_fd, buf, len<4096?len:4096, flags);/g' | sed 's/ sizeof(addr->sa_data)/ 28/g' >$(PYDIR)/Modules/socketmodule.c - #cp $(PYDIR)/Objects/exceptions.c $(PYDIR)/Objects/exceptions.c_old - #cat $(PYDIR)/Objects/exceptions.c_old | sed 's/ESHUTDOWN/110/g' >$(PYDIR)/Objects/exceptions.c - #cp $(PYDIR)/Python/pytime.c $(PYDIR)/Python/pytime.c_old - #cat $(PYDIR)/Python/pytime.c_old | sed 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c + cp $(PYDIR)/Objects/exceptions.c $(PYDIR)/Objects/exceptions.c_old + cat $(PYDIR)/Objects/exceptions.c_old | sed 's/ESHUTDOWN/EPIPE/g' >$(PYDIR)/Objects/exceptions.c + cp $(PYDIR)/Python/pytime.c $(PYDIR)/Python/pytime.c_old + cat $(PYDIR)/Python/pytime.c_old | sed 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c cp $(PYDIR)/Makefile $(PYDIR)/Makefile_old cat $(PYDIR)/Makefile_old | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile touch soospatchPY diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index a8ce01d..97bb9f3 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -13,15 +13,6 @@ support for AIX C++ shared extension modules. */ /* #undef AIX_GENUINE_CPLUSPLUS */ -/* Define this if you have AtheOS threads. */ -/* #undef ATHEOS_THREADS */ - -/* Define this if you have BeOS threads. */ -/* #undef BEOS_THREADS */ - -/* Define if you have the Mach cthreads package */ -/* #undef C_THREADS */ - /* Define if C doubles are 64-bit IEEE 754 binary format, stored in ARM mixed-endian order (byte order 45670123) */ /* #undef DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754 */ @@ -35,7 +26,7 @@ /* #undef DOUBLE_IS_LITTLE_ENDIAN_IEEE754 */ /* Define if --enable-ipv6 is specified */ -#undef ENABLE_IPV6 +/* #undef ENABLE_IPV6 */ /* Define if flock needs to be linked with bsd library. */ /* #undef FLOCK_NEEDS_LIBBSD */ @@ -47,15 +38,20 @@ the case on Motorola V4 (R40V4.2) */ /* #undef GETTIMEOFDAY_NO_TZ */ +/* Define to 1 if you have the `accept4' function. */ +#define HAVE_ACCEPT4 1 + /* Define to 1 if you have the `acosh' function. */ #define HAVE_ACOSH 1 /* struct addrinfo (netdb.h) */ -/* #undef HAVE_ADDRINFO */ -#define HAVE_ADDRINFO 1 +#define HAVE_ADDRINFO 1 // original: /* #undef HAVE_ADDRINFO */ /* Define to 1 if you have the `alarm' function. */ -#undef HAVE_ALARM +#undef HAVE_ALARM // original: #define HAVE_ALARM 1 + +/* Define if aligned memory access is required */ +#define HAVE_ALIGNED_REQUIRED 1 /* Define to 1 if you have the header file. */ #define HAVE_ALLOCA_H 1 @@ -72,9 +68,6 @@ /* Define to 1 if you have the `atanh' function. */ #define HAVE_ATANH 1 -/* Define if GCC supports __attribute__((format(PyArg_ParseTuple, 2, 3))) */ -/* #undef HAVE_ATTRIBUTE_FORMAT_PARSETUPLE */ - /* Define to 1 if you have the `bind_textdomain_codeset' function. */ #define HAVE_BIND_TEXTDOMAIN_CODESET 1 @@ -84,6 +77,10 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_BLUETOOTH_H */ +/* Define if mbstowcs(NULL, "text", 0) does not return the number of wide + chars that would be converted. */ +/* #undef HAVE_BROKEN_MBSTOWCS */ + /* Define if nice() returns success/failure instead of the new priority. */ /* #undef HAVE_BROKEN_NICE */ @@ -105,26 +102,35 @@ /* Define if `unsetenv` does not return an int. */ /* #undef HAVE_BROKEN_UNSETENV */ +/* Has builtin atomics */ +#define HAVE_BUILTIN_ATOMIC 1 + /* Define this if you have the type _Bool. */ #define HAVE_C99_BOOL 1 /* Define to 1 if you have the 'chflags' function. */ -/* #define HAVE_CHFLAGS 1 */ +/* #define HAVE_CHFLAGS 1 */ // original: #define HAVE_CHFLAGS 1 /* Define to 1 if you have the `chown' function. */ -/* #define HAVE_CHOWN 1 */ +/* #define HAVE_CHOWN 1 */ // original: #define HAVE_CHOWN 1 /* Define if you have the 'chroot' function. */ -/* #define HAVE_CHROOT 1 */ +/* #define HAVE_CHROOT 1 */ // original: #define HAVE_CHROOT 1 /* Define to 1 if you have the `clock' function. */ #define HAVE_CLOCK 1 +/* Define to 1 if you have the `clock_getres' function. */ +#define HAVE_CLOCK_GETRES 1 + +/* Define to 1 if you have the `clock_gettime' function. */ +#define HAVE_CLOCK_GETTIME 1 + /* Define if the C compiler supports computed gotos. */ /* #undef HAVE_COMPUTED_GOTOS */ /* Define to 1 if you have the `confstr' function. */ -/* #define HAVE_CONFSTR 1 */ +/* #define HAVE_CONFSTR 1 */ // original: #define HAVE_CONFSTR 1 /* Define to 1 if you have the header file. */ /* #undef HAVE_CONIO_H */ @@ -133,7 +139,7 @@ #define HAVE_COPYSIGN 1 /* Define to 1 if you have the `ctermid' function. */ -/* #define HAVE_CTERMID 1 */ +/* #define HAVE_CTERMID 1 */ // original: #define HAVE_CTERMID 1 /* Define if you have the 'ctermid_r' function. */ /* #undef HAVE_CTERMID_R */ @@ -164,10 +170,10 @@ /* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. */ -#undef HAVE_DECL_TZNAME +#undef HAVE_DECL_TZNAME // original: #define HAVE_DECL_TZNAME 1 /* Define to 1 if you have the device macros. */ -/* #define HAVE_DEVICE_MACROS 1 */ +/* #define HAVE_DEVICE_MACROS 1 */ // original: #define HAVE_DEVICE_MACROS 1 /* Define to 1 if you have the /dev/ptc device file. */ /* #undef HAVE_DEV_PTC */ @@ -178,25 +184,40 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_DIRECT_H */ +/* Define to 1 if the dirent structure has a d_type field */ +#define HAVE_DIRENT_D_TYPE 1 + /* Define to 1 if you have the header file, and it defines `DIR'. */ #define HAVE_DIRENT_H 1 +/* Define if you have the 'dirfd' function or macro. */ +/* #undef HAVE_DIRFD */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_DLFCN_H */ /* Define to 1 if you have the `dlopen' function. */ -#undef HAVE_DLOPEN +#undef HAVE_DLOPEN // original: #define HAVE_DLOPEN 1 /* Define to 1 if you have the `dup2' function. */ #define HAVE_DUP2 1 +/* Define to 1 if you have the `dup3' function. */ +#define HAVE_DUP3 1 + /* Defined when any dynamic module loading is enabled. */ -#undef HAVE_DYNAMIC_LOADING +#undef HAVE_DYNAMIC_LOADING // original: #define HAVE_DYNAMIC_LOADING 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_ENDIAN_H */ /* Define if you have the 'epoll' functions. */ /* #undef HAVE_EPOLL */ +/* Define if you have the 'epoll_create1' function. */ +/* #undef HAVE_EPOLL_CREATE1 */ + /* Define to 1 if you have the `erf' function. */ #define HAVE_ERF 1 @@ -207,25 +228,40 @@ #define HAVE_ERRNO_H 1 /* Define to 1 if you have the `execv' function. */ -#undef HAVE_EXECV +#undef HAVE_EXECV // original: #define HAVE_EXECV 1 /* Define to 1 if you have the `expm1' function. */ #define HAVE_EXPM1 1 +/* Define to 1 if you have the `faccessat' function. */ +#define HAVE_FACCESSAT 1 + /* Define if you have the 'fchdir' function. */ -/* #define HAVE_FCHDIR 1 */ +/* #define HAVE_FCHDIR 1 */ // #define HAVE_FCHDIR 1 /* Define to 1 if you have the `fchmod' function. */ #define HAVE_FCHMOD 1 +/* Define to 1 if you have the `fchmodat' function. */ +#define HAVE_FCHMODAT 1 + /* Define to 1 if you have the `fchown' function. */ -/* #define HAVE_FCHOWN 1 */ +#define HAVE_FCHOWN 1 + +/* Define to 1 if you have the `fchownat' function. */ +#define HAVE_FCHOWNAT 1 /* Define to 1 if you have the header file. */ #define HAVE_FCNTL_H 1 /* Define if you have the 'fdatasync' function. */ -#undef HAVE_FDATASYNC +#undef HAVE_FDATASYNC // original: #define HAVE_FDATASYNC 1 + +/* Define to 1 if you have the `fdopendir' function. */ +#define HAVE_FDOPENDIR 1 + +/* Define to 1 if you have the `fexecve' function. */ +#define HAVE_FEXECVE 1 /* Define to 1 if you have the `finite' function. */ #define HAVE_FINITE 1 @@ -234,13 +270,13 @@ #define HAVE_FLOCK 1 /* Define to 1 if you have the `fork' function. */ -#undef HAVE_FORK +#undef HAVE_FORK // original: #define HAVE_FORK 1 /* Define to 1 if you have the `forkpty' function. */ -#undef HAVE_FORK +#undef HAVE_FORKPTY // original: #define HAVE_FORKPTY 1 /* Define to 1 if you have the `fpathconf' function. */ -/* #define HAVE_FPATHCONF 1 */ +/* #define HAVE_FPATHCONF 1 */ // original: #define HAVE_FPATHCONF 1 /* Define to 1 if you have the `fseek64' function. */ #define HAVE_FSEEK64 1 @@ -248,6 +284,9 @@ /* Define to 1 if you have the `fseeko' function. */ #define HAVE_FSEEKO 1 +/* Define to 1 if you have the `fstatat' function. */ +#define HAVE_FSTATAT 1 + /* Define to 1 if you have the `fstatvfs' function. */ #define HAVE_FSTATVFS 1 @@ -261,42 +300,60 @@ #define HAVE_FTELLO 1 /* Define to 1 if you have the `ftime' function. */ -/* #define HAVE_FTIME 1 */ +/* #define HAVE_FTIME 1 */ // original: #define HAVE_FTIME 1 /* Define to 1 if you have the `ftruncate' function. */ #define HAVE_FTRUNCATE 1 +/* Define to 1 if you have the `futimens' function. */ +#define HAVE_FUTIMENS 1 + +/* Define to 1 if you have the `futimes' function. */ +#define HAVE_FUTIMES 1 + +/* Define to 1 if you have the `futimesat' function. */ +#define HAVE_FUTIMESAT 1 + /* Define to 1 if you have the `gai_strerror' function. */ #define HAVE_GAI_STRERROR 1 /* Define to 1 if you have the `gamma' function. */ #define HAVE_GAMMA 1 +/* Define if we can use gcc inline assembler to get and set mc68881 fpcr */ +/* #undef HAVE_GCC_ASM_FOR_MC68881 */ + +/* Define if we can use x64 gcc inline assembler */ +/* #undef HAVE_GCC_ASM_FOR_X64 */ + /* Define if we can use gcc inline assembler to get and set x87 control word */ /* #undef HAVE_GCC_ASM_FOR_X87 */ -/* Define if you have the getaddrinfo function. */ +/* Define if your compiler provides __uint128_t */ +#define HAVE_GCC_UINT128_T 1 +/* Define if you have the getaddrinfo function. */ #define HAVE_GETADDRINFO 1 -/* struct servent {char*s_name;char**s_aliases;int s_port;char*s_proto;}; - struct protoent{char*p_name;char**p_aliases;int p_proto;}; */ +// TODO clarify what these are #define SOCK_RAW 3 + #define SOCK_SEQPACKET 5 -#define IN_CLASSA_NSHIFT 24/* #undef HAVE_GETADDRINFO */ -/* Define to 1 if you have the `getcwd' function. */ -#define HAVE_GETCWD 1 +#define IN_CLASSA_NSHIFT 24 /* Define this if you have flockfile(), getc_unlocked(), and funlockfile() */ -#undef HAVE_GETC_UNLOCKED +#undef HAVE_GETC_UNLOCKED // original: #define HAVE_GETC_UNLOCKED 1 /* Define to 1 if you have the `getentropy' function. */ -#undef HAVE_GETENTROPY +#undef HAVE_GETENTROPY // original: #define HAVE_GETENTROPY 1 + +/* Define to 1 if you have the `getgrouplist' function. */ +#define HAVE_GETGROUPLIST 1 /* Define to 1 if you have the `getgroups' function. */ -/* #define HAVE_GETGROUPS 1 */ +/* #define HAVE_GETGROUPS 1 */ // original: #define HAVE_GETGROUPS 1 /* Define to 1 if you have the `gethostbyname' function. */ /* #undef HAVE_GETHOSTBYNAME */ @@ -314,13 +371,13 @@ /* #undef HAVE_GETHOSTBYNAME_R_6_ARG */ /* Define to 1 if you have the `getitimer' function. */ -#undef HAVE_GETITIMER +#undef HAVE_GETITIMER // original: #define HAVE_GETITIMER 1 /* Define to 1 if you have the `getloadavg' function. */ -/* #define HAVE_GETLOADAVG 1 */ +/* #define HAVE_GETLOADAVG 1 */ // original: #define HAVE_GETLOADAVG 1 /* Define to 1 if you have the `getlogin' function. */ -/* #define HAVE_GETLOGIN 1 */ +/* #define HAVE_GETLOGIN 1 */ // original: #define HAVE_GETLOGIN 1 /* Define to 1 if you have the `getnameinfo' function. */ #define HAVE_GETNAMEINFO 1 @@ -332,10 +389,10 @@ #define HAVE_GETPEERNAME 1 /* Define to 1 if you have the `getpgid' function. */ -/* #define HAVE_GETPGID 1 */ +/* #define HAVE_GETPGID 1 */ // original: #define HAVE_GETPGID 1 /* Define to 1 if you have the `getpgrp' function. */ -/* #define HAVE_GETPGRP 1 */ +/* #define HAVE_GETPGRP 1 */ // original: #define HAVE_GETPGRP 1 /* Define to 1 if you have the `getpid' function. */ #define HAVE_GETPID 1 @@ -344,16 +401,22 @@ #define HAVE_GETPRIORITY 1 /* Define to 1 if you have the `getpwent' function. */ -#undef HAVE_GETPWENT +#undef HAVE_GETPWENT // original: #define HAVE_GETPWENT 1 + +/* Define to 1 if the getrandom() function is available */ +/* #undef HAVE_GETRANDOM */ + +/* Define to 1 if the Linux getrandom() syscall is available */ +/* #undef HAVE_GETRANDOM_SYSCALL */ /* Define to 1 if you have the `getresgid' function. */ -/* #define HAVE_GETRESGID 1 */ +/* #define HAVE_GETRESGID 1 */ // original: #define HAVE_GETRESGID 1 /* Define to 1 if you have the `getresuid' function. */ -/* #define HAVE_GETRESUID 1 */ +/* #define HAVE_GETRESUID 1 */ // original: #define HAVE_GETRESUID 1 /* Define to 1 if you have the `getsid' function. */ -/* #define HAVE_GETSID 1 */ +/* #define HAVE_GETSID 1 */ // original: #define HAVE_GETSID 1 /* Define to 1 if you have the `getspent' function. */ #define HAVE_GETSPENT 1 @@ -367,27 +430,36 @@ /* Define to 1 if you have the `getwd' function. */ #define HAVE_GETWD 1 +/* Define if glibc has incorrect _FORTIFY_SOURCE wrappers for memmove and + bcopy. */ +/* #undef HAVE_GLIBC_MEMMOVE_BUG */ + /* Define to 1 if you have the header file. */ #define HAVE_GRP_H 1 /* Define if you have the 'hstrerror' function. */ /* #undef HAVE_HSTRERROR */ +/* Define this if you have le64toh() */ +#define HAVE_HTOLE64 1 + /* Define to 1 if you have the `hypot' function. */ #define HAVE_HYPOT 1 /* Define to 1 if you have the header file. */ #define HAVE_IEEEFP_H 1 +/* Define to 1 if you have the `if_nameindex' function. */ +#define HAVE_IF_NAMEINDEX 1 + /* Define if you have the 'inet_aton' function. */ -/* #undef HAVE_INET_ATON */ +/* #define HAVE_INET_ATON 1 */ // original: #define HAVE_INET_ATON 1 /* Define if you have the 'inet_pton' function. */ -/* #undef HAVE_INET_PTON */ #define HAVE_INET_PTON 1 /* Define to 1 if you have the `initgroups' function. */ -/* #define HAVE_INITGROUPS 1 */ +/* #define HAVE_INITGROUPS 1 */ // original: #define HAVE_INITGROUPS 1 /* Define if your compiler provides int32_t. */ #define HAVE_INT32_T 1 @@ -401,11 +473,14 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_IO_H */ +/* Define if gcc has the ipa-pure-const bug. */ +/* #undef HAVE_IPA_PURE_CONST_BUG */ + /* Define to 1 if you have the `kill' function. */ -#undef HAVE_KILL +#undef HAVE_KILL // original: #define HAVE_KILL 1 /* Define to 1 if you have the `killpg' function. */ -#undef HAVE_KILL +#undef HAVE_KILLPG // original: #define HAVE_KILLPG 1 /* Define if you have the 'kqueue' functions. */ /* #undef HAVE_KQUEUE */ @@ -420,13 +495,13 @@ /* #undef HAVE_LARGEFILE_SUPPORT */ /* Define to 1 if you have the 'lchflags' function. */ -/* #define HAVE_LCHFLAGS 1 */ +/* #define HAVE_LCHFLAGS 1 */ // original: #define HAVE_LCHFLAGS 1 /* Define to 1 if you have the `lchmod' function. */ -/* #define HAVE_LCHMOD 1 */ +/* #define HAVE_LCHMOD 1 */ // original: #define HAVE_LCHMOD 1 /* Define to 1 if you have the `lchown' function. */ -/* #define HAVE_LCHOWN 1 */ +/* #define HAVE_LCHOWN 1 */ // original: #define HAVE_LCHOWN 1 /* Define to 1 if you have the `lgamma' function. */ #define HAVE_LGAMMA 1 @@ -449,21 +524,48 @@ /* Define to 1 if you have the `resolv' library (-lresolv). */ /* #undef HAVE_LIBRESOLV */ +/* Define to 1 if you have the `sendfile' library (-lsendfile). */ +/* #undef HAVE_LIBSENDFILE */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_LIBUTIL_H */ /* Define if you have the 'link' function. */ #define HAVE_LINK 1 +/* Define to 1 if you have the `linkat' function. */ +#define HAVE_LINKAT 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LINUX_CAN_BCM_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LINUX_CAN_H */ + +/* Define if compiling using Linux 3.6 or later. */ +/* #undef HAVE_LINUX_CAN_RAW_FD_FRAMES */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LINUX_CAN_RAW_H */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_LINUX_NETLINK_H */ +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LINUX_RANDOM_H */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_LINUX_TIPC_H */ +/* Define to 1 if you have the `lockf' function. */ +#define HAVE_LOCKF 1 + /* Define to 1 if you have the `log1p' function. */ #define HAVE_LOG1P 1 +/* Define to 1 if you have the `log2' function. */ +#define HAVE_LOG2 1 + /* Define this if you have the type long double. */ #define HAVE_LONG_DOUBLE 1 @@ -471,10 +573,16 @@ #define HAVE_LONG_LONG 1 /* Define to 1 if you have the `lstat' function. */ -/* #define HAVE_LSTAT 1 */ +/* #define HAVE_LSTAT 1 */ // original: #define HAVE_LSTAT 1 + +/* Define to 1 if you have the `lutimes' function. */ +#define HAVE_LUTIMES 1 /* Define this if you have the makedev macro. */ -/* #define HAVE_MAKEDEV 1 */ +/* #define HAVE_MAKEDEV 1 */ // original: #define HAVE_MAKEDEV 1 + +/* Define to 1 if you have the `mbrtowc' function. */ +#define HAVE_MBRTOWC 1 /* Define to 1 if you have the `memmove' function. */ #define HAVE_MEMMOVE 1 @@ -482,17 +590,29 @@ /* Define to 1 if you have the header file. */ #define HAVE_MEMORY_H 1 +/* Define to 1 if you have the `memrchr' function. */ +#define HAVE_MEMRCHR 1 + +/* Define to 1 if you have the `mkdirat' function. */ +#define HAVE_MKDIRAT 1 + /* Define to 1 if you have the `mkfifo' function. */ -/* #define HAVE_MKFIFO 1 */ +/* #define HAVE_MKFIFO 1 */ // original: #define HAVE_MKFIFO 1 + +/* Define to 1 if you have the `mkfifoat' function. */ +#define HAVE_MKFIFOAT 1 /* Define to 1 if you have the `mknod' function. */ -/* #define HAVE_MKNOD 1 */ +/* #define HAVE_MKNOD 1 */ // original: #define HAVE_MKNOD 1 + +/* Define to 1 if you have the `mknodat' function. */ +#define HAVE_MKNODAT 1 /* Define to 1 if you have the `mktime' function. */ #define HAVE_MKTIME 1 /* Define to 1 if you have the `mmap' function. */ -#undef HAVE_MMAP +#undef HAVE_MMAP // original: #define HAVE_MMAP 1 /* Define to 1 if you have the `mremap' function. */ #define HAVE_MREMAP 1 @@ -506,29 +626,50 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_NETPACKET_PACKET_H */ +/* Define to 1 if you have the header file. */ +#define HAVE_NET_IF_H 1 + /* Define to 1 if you have the `nice' function. */ -/* #define HAVE_NICE 1 */ +/* #define HAVE_NICE 1 */ // original: #define HAVE_NICE 1 + +/* Define to 1 if you have the `openat' function. */ +#define HAVE_OPENAT 1 /* Define to 1 if you have the `openpty' function. */ -#undef HAVE_OPENPTY +#undef HAVE_OPENPTY // original: #define HAVE_OPENPTY 1 /* Define if compiling using MacOS X 10.5 SDK or later. */ /* #undef HAVE_OSX105_SDK */ /* Define to 1 if you have the `pathconf' function. */ -/* #define HAVE_PATHCONF 1 */ +/* #define HAVE_PATHCONF 1 */ // original: #define HAVE_PATHCONF 1 /* Define to 1 if you have the `pause' function. */ -#undef HAVE_PAUSE +#undef HAVE_PAUSE // original: #define HAVE_PAUSE 1 + +/* Define to 1 if you have the `pipe2' function. */ +#define HAVE_PIPE2 1 /* Define to 1 if you have the `plock' function. */ -/* #define HAVE_PLOCK 1 */ +/* #define HAVE_PLOCK 1 */ // original: #define HAVE_PLOCK 1 /* Define to 1 if you have the `poll' function. */ -/* #define HAVE_POLL 1 */ +/* #define HAVE_POLL 1 */ // original: #define HAVE_POLL 1 /* Define to 1 if you have the header file. */ -/* #undef HAVE_POLL_H */ +/* #undef HAVE_POLL_H */ // original: #define HAVE_POLL_H 1 + +/* Define to 1 if you have the `posix_fadvise' function. */ +#define HAVE_POSIX_FADVISE 1 + +/* Define to 1 if you have the `posix_fallocate' function. */ +#define HAVE_POSIX_FALLOCATE 1 + +/* Define to 1 if you have the `pread' function. */ +#define HAVE_PREAD 1 + +/* Define if you have the 'prlimit' functions. */ +/* #undef HAVE_PRLIMIT */ /* Define to 1 if you have the header file. */ /* #undef HAVE_PROCESS_H */ @@ -536,9 +677,6 @@ /* Define if your compiler supports function prototype */ #define HAVE_PROTOTYPES 1 -/* Define if you have GNU PTH threads. */ -/* #undef HAVE_PTH */ - /* Define to 1 if you have the `pthread_atfork' function. */ /* #undef HAVE_PTHREAD_ATFORK */ @@ -551,6 +689,9 @@ /* Define to 1 if you have the `pthread_init' function. */ #define HAVE_PTHREAD_INIT 1 +/* Define to 1 if you have the `pthread_kill' function. */ +#define HAVE_PTHREAD_KILL 1 + /* Define to 1 if you have the `pthread_sigmask' function. */ /* #undef HAVE_PTHREAD_SIGMASK */ @@ -560,15 +701,27 @@ /* Define to 1 if you have the `putenv' function. */ #define HAVE_PUTENV 1 -/* Define if the libcrypto has RAND_egd */ -/* #undef HAVE_RAND_EGD */ +/* Define to 1 if you have the `pwrite' function. */ +#define HAVE_PWRITE 1 /* Define to 1 if you have the `readlink' function. */ -#undef HAVE_READLINK +#undef HAVE_READLINK // original: #define HAVE_READLINK 1 + +/* Define to 1 if you have the `readlinkat' function. */ +#define HAVE_READLINKAT 1 + +/* Define to 1 if you have the `readv' function. */ +#define HAVE_READV 1 /* Define to 1 if you have the `realpath' function. */ #define HAVE_REALPATH 1 +/* Define to 1 if you have the `renameat' function. */ +#define HAVE_RENAMEAT 1 + +/* Define if readline supports append_history */ +/* #undef HAVE_RL_APPEND_HISTORY */ + /* Define if you have readline 2.1 */ /* #undef HAVE_RL_CALLBACK */ @@ -596,6 +749,24 @@ /* Define to 1 if you have the `round' function. */ #define HAVE_ROUND 1 +/* Define to 1 if you have the `sched_get_priority_max' function. */ +#define HAVE_SCHED_GET_PRIORITY_MAX 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SCHED_H 1 + +/* Define to 1 if you have the `sched_rr_get_interval' function. */ +#define HAVE_SCHED_RR_GET_INTERVAL 1 + +/* Define to 1 if you have the `sched_setaffinity' function. */ +#define HAVE_SCHED_SETAFFINITY 1 + +/* Define to 1 if you have the `sched_setparam' function. */ +#define HAVE_SCHED_SETPARAM 1 + +/* Define to 1 if you have the `sched_setscheduler' function. */ +#define HAVE_SCHED_SETSCHEDULER 1 + /* Define to 1 if you have the `select' function. */ #define HAVE_SELECT 1 @@ -611,47 +782,56 @@ /* Define to 1 if you have the `sem_unlink' function. */ #define HAVE_SEM_UNLINK 1 +/* Define to 1 if you have the `sendfile' function. */ +#define HAVE_SENDFILE 1 + /* Define to 1 if you have the `setegid' function. */ -/* #define HAVE_SETEGID 1 */ +/* #define HAVE_SETEGID 1 */ // original: #define HAVE_SETEGID 1 /* Define to 1 if you have the `seteuid' function. */ -/* #define HAVE_SETEUID 1 */ +/* #define HAVE_SETEUID 1 */ // original: #define HAVE_SETEUID 1 /* Define to 1 if you have the `setgid' function. */ -/* #define HAVE_SETGID 1 */ +/* #define HAVE_SETGID 1 */ // original: #define HAVE_SETGID 1 /* Define if you have the 'setgroups' function. */ -/* #define HAVE_SETGROUPS 1 */ +/* #define HAVE_SETGROUPS 1 */ // original: #define HAVE_SETGROUPS 1 + +/* Define to 1 if you have the `sethostname' function. */ +#define HAVE_SETHOSTNAME 1 /* Define to 1 if you have the `setitimer' function. */ -/* #define HAVE_SETITIMER 1 */ +/* #define HAVE_SETITIMER 1 */ // original: #define HAVE_SETITIMER 1 /* Define to 1 if you have the `setlocale' function. */ #define HAVE_SETLOCALE 1 /* Define to 1 if you have the `setpgid' function. */ -/* #define HAVE_SETPGID 1 */ +/* #define HAVE_SETPGID 1 */ // original: #define HAVE_SETPGID 1 /* Define to 1 if you have the `setpgrp' function. */ -/* #define HAVE_SETPGRP 1 */ +/* #define HAVE_SETPGRP 1 */ // original: #define HAVE_SETPGRP 1 + +/* Define to 1 if you have the `setpriority' function. */ +#define HAVE_SETPRIORITY 1 /* Define to 1 if you have the `setregid' function. */ -/* #define HAVE_SETREGID 1 */ +/* #define HAVE_SETREGID 1 */ // original: #define HAVE_SETREGID 1 /* Define to 1 if you have the `setresgid' function. */ -/* #define HAVE_SETRESGID 1 */ +/* #define HAVE_SETRESGID 1 */ // original: #define HAVE_SETRESGID 1 /* Define to 1 if you have the `setresuid' function. */ -/* #define HAVE_SETRESUID 1 */ +/* #define HAVE_SETRESUID 1 */ // original: #define HAVE_SETRESUID 1 /* Define to 1 if you have the `setreuid' function. */ -/* #define HAVE_SETREUID 1 */ +/* #define HAVE_SETREUID 1 */ // original: #define HAVE_SETREUID 1 /* Define to 1 if you have the `setsid' function. */ -/* #define HAVE_SETSID 1 */ +/* #define HAVE_SETSID 1 */ // original: #define HAVE_SETSID 1 /* Define to 1 if you have the `setuid' function. */ -/* #define HAVE_SETUID 1 */ +/* #define HAVE_SETUID 1 */ // original: #define HAVE_SETUID 1 /* Define to 1 if you have the `setvbuf' function. */ #define HAVE_SETVBUF 1 @@ -660,29 +840,43 @@ /* #undef HAVE_SHADOW_H */ /* Define to 1 if you have the `sigaction' function. */ -#undef HAVE_SIGACTION +#undef HAVE_SIGACTION // original: #define HAVE_SIGACTION 1 + +/* Define to 1 if you have the `sigaltstack' function. */ +#define HAVE_SIGALTSTACK 1 /* Define to 1 if you have the `siginterrupt' function. */ -#undef HAVE_SIGINTERRUPT +#undef HAVE_SIGINTERRUPT // original: #define HAVE_SIGINTERRUPT 1 /* Define to 1 if you have the header file. */ #define HAVE_SIGNAL_H 1 +/* Define to 1 if you have the `sigpending' function. */ +#define HAVE_SIGPENDING 1 + /* Define to 1 if you have the `sigrelse' function. */ #define HAVE_SIGRELSE 1 +/* Define to 1 if you have the `sigtimedwait' function. */ +#define HAVE_SIGTIMEDWAIT 1 + +/* Define to 1 if you have the `sigwait' function. */ +#define HAVE_SIGWAIT 1 + +/* Define to 1 if you have the `sigwaitinfo' function. */ +#define HAVE_SIGWAITINFO 1 + /* Define to 1 if you have the `snprintf' function. */ #define HAVE_SNPRINTF 1 /* Define if sockaddr has sa_len member */ -/* #undef HAVE_SOCKADDR_SA_LEN */ +/* #define HAVE_SOCKADDR_SA_LEN 1 */ // original: #define HAVE_SOCKADDR_SA_LEN 1 /* struct sockaddr_storage (sys/socket.h) */ -/* #undef HAVE_SOCKADDR_STORAGE */ #define HAVE_SOCKADDR_STORAGE 1 /* Define if you have the 'socketpair' function. */ -/* #undef HAVE_SOCKETPAIR */ +/* #define HAVE_SOCKETPAIR 1 */ // original: #define HAVE_SOCKETPAIR 1 /* Define to 1 if you have the header file. */ #define HAVE_SPAWN_H 1 @@ -691,7 +885,7 @@ #define HAVE_SSIZE_T 1 /* Define to 1 if you have the `statvfs' function. */ -#undef HAVE_STATVFS +#undef HAVE_STATVFS // original: #define HAVE_STATVFS 1 /* Define if you have struct stat.st_mtim.tv_nsec */ /* #undef HAVE_STAT_TV_NSEC */ @@ -709,6 +903,9 @@ /* Define to 1 if you have the header file. */ #define HAVE_STDLIB_H 1 +/* Has stdatomic.h, atomic_int and _Atomic void* types work */ +#define HAVE_STD_ATOMIC 1 + /* Define to 1 if you have the `strdup' function. */ #define HAVE_STRDUP 1 @@ -721,6 +918,9 @@ /* Define to 1 if you have the header file. */ #define HAVE_STRING_H 1 +/* Define to 1 if you have the `strlcpy' function. */ +#define HAVE_STRLCPY 1 + /* Define to 1 if you have the header file. */ /* #undef HAVE_STROPTS_H */ @@ -746,10 +946,16 @@ /* #undef HAVE_STRUCT_TM_TM_ZONE */ /* Define if you have the 'symlink' function. */ -#undef HAVE_SYMLINK +#undef HAVE_SYMLINK // original: #define HAVE_SYMLINK 1 + +/* Define to 1 if you have the `symlinkat' function. */ +#define HAVE_SYMLINKAT 1 + +/* Define to 1 if you have the `sync' function. */ +#define HAVE_SYNC 1 /* Define to 1 if you have the `sysconf' function. */ -/* #define HAVE_SYSCONF 1 */ +/* #define HAVE_SYSCONF 1 */ // original: #define HAVE_SYSCONF 1 /* Define to 1 if you have the header file. */ /* #undef HAVE_SYSEXITS_H */ @@ -760,10 +966,16 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_BSDTTY_H */ +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DEVPOLL_H */ + /* Define to 1 if you have the header file, and it defines `DIR'. */ /* #undef HAVE_SYS_DIR_H */ +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_ENDIAN_H */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_EPOLL_H */ @@ -773,14 +985,20 @@ /* Define to 1 if you have the header file. */ #define HAVE_SYS_FILE_H 1 +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_IOCTL_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_KERN_CONTROL_H */ + /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_LOADAVG_H +#undef HAVE_SYS_LOADAVG_H // original: /* #undef HAVE_SYS_LOADAVG_H */ /* Define to 1 if you have the header file. */ #define HAVE_SYS_LOCK_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_MKDEV_H +#undef HAVE_SYS_MKDEV_H // original: /* #undef HAVE_SYS_MKDEV_H */ /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_MODEM_H */ @@ -795,21 +1013,33 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_POLL_H */ +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_RANDOM_H */ + /* Define to 1 if you have the header file. */ #define HAVE_SYS_RESOURCE_H 1 /* Define to 1 if you have the header file. */ #define HAVE_SYS_SELECT_H 1 +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_SENDFILE_H */ + /* Define to 1 if you have the header file. */ -/* #undef HAVE_SYS_SOCKET_H */ +/* #define HAVE_SYS_SOCKET_H 1 */ // original: #define HAVE_SYS_SOCKET_H 1 /* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STATVFS_H +#undef HAVE_SYS_STATVFS_H // original: #define HAVE_SYS_STATVFS_H 1 /* Define to 1 if you have the header file. */ #define HAVE_SYS_STAT_H 1 +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_SYSCALL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_SYS_DOMAIN_H */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_TERMIO_H */ @@ -822,6 +1052,9 @@ /* Define to 1 if you have the header file. */ #define HAVE_SYS_TYPES_H 1 +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_UIO_H */ + /* Define to 1 if you have the header file. */ /* #undef HAVE_SYS_UN_H */ @@ -831,11 +1064,14 @@ /* Define to 1 if you have the header file. */ #define HAVE_SYS_WAIT_H 1 +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_XATTR_H */ + /* Define to 1 if you have the `tcgetpgrp' function. */ -/* #define HAVE_TCGETPGRP 1 */ +/* #define HAVE_TCGETPGRP 1 */ // original: #define HAVE_TCGETPGRP 1 /* Define to 1 if you have the `tcsetpgrp' function. */ -/* #define HAVE_TCSETPGRP 1 */ +/* #define HAVE_TCSETPGRP 1 */ // original: #define HAVE_TCSETPGRP 1 /* Define to 1 if you have the `tempnam' function. */ #define HAVE_TEMPNAM 1 @@ -849,9 +1085,6 @@ /* Define to 1 if you have the `tgamma' function. */ #define HAVE_TGAMMA 1 -/* Define to 1 if you have the header file. */ -/* #undef HAVE_THREAD_H */ - /* Define to 1 if you have the `timegm' function. */ #define HAVE_TIMEGM 1 @@ -876,7 +1109,7 @@ /* Define to 1 if you don't have `tm_zone' but do have the external array `tzname'. */ -#undef HAVE_TZNAME +#undef HAVE_TZNAME // original: #define HAVE_TZNAME 1 /* Define this if you have tcl and TCL_UTF_MAX==6 */ /* #undef HAVE_UCS4_TCL */ @@ -891,11 +1124,14 @@ #define HAVE_UINTPTR_T 1 /* Define to 1 if you have the `uname' function. */ -/* #define HAVE_UNAME 1 */ +/* #define HAVE_UNAME 1 */ // original: #define HAVE_UNAME 1 /* Define to 1 if you have the header file. */ #define HAVE_UNISTD_H 1 +/* Define to 1 if you have the `unlinkat' function. */ +#define HAVE_UNLINKAT 1 + /* Define to 1 if you have the `unsetenv' function. */ #define HAVE_UNSETENV 1 @@ -907,20 +1143,26 @@ /* Define to 1 if you have the header file. */ /* #undef HAVE_UTIL_H */ +/* Define to 1 if you have the `utimensat' function. */ +#define HAVE_UTIMENSAT 1 + /* Define to 1 if you have the `utimes' function. */ #define HAVE_UTIMES 1 /* Define to 1 if you have the header file. */ -/* #undef HAVE_UTIME_H */ +/* #define HAVE_UTIME_H 1 */ // original: #define HAVE_UTIME_H 1 /* Define to 1 if you have the `wait3' function. */ -/* #define HAVE_WAIT3 1 */ +/* #define HAVE_WAIT3 1 */ // original: #define HAVE_WAIT3 1 /* Define to 1 if you have the `wait4' function. */ -/* #define HAVE_WAIT4 1 */ +/* #define HAVE_WAIT4 1 */ // original: #define HAVE_WAIT4 1 + +/* Define to 1 if you have the `waitid' function. */ +#define HAVE_WAITID 1 /* Define to 1 if you have the `waitpid' function. */ -/* #define HAVE_WAITPID 1 */ +/* #define HAVE_WAITPID 1 */ // original: #define HAVE_WAITPID 1 /* Define if the compiler provides a wchar.h header file. */ #define HAVE_WCHAR_H 1 @@ -928,25 +1170,34 @@ /* Define to 1 if you have the `wcscoll' function. */ #define HAVE_WCSCOLL 1 +/* Define to 1 if you have the `wcsftime' function. */ +#define HAVE_WCSFTIME 1 + +/* Define to 1 if you have the `wcsxfrm' function. */ +#define HAVE_WCSXFRM 1 + +/* Define to 1 if you have the `wmemcmp' function. */ +#define HAVE_WMEMCMP 1 + /* Define if tzset() actually switches the local timezone in a meaningful way. */ /* #undef HAVE_WORKING_TZSET */ +/* Define to 1 if you have the `writev' function. */ +#define HAVE_WRITEV 1 + /* Define if the zlib library has inflateCopy */ /* #undef HAVE_ZLIB_COPY */ /* Define to 1 if you have the `_getpty' function. */ -/* #define HAVE__GETPTY 1 */ - -/* Define if you are using Mach cthreads directly under /include */ -/* #undef HURD_C_THREADS */ +/* #define HAVE__GETPTY 1 */ // original: #define HAVE__GETPTY 1 -/* Define if you are using Mach cthreads under mach / */ -/* #undef MACH_C_THREADS */ +/* Define if log1p(-0.) is 0. rather than -0. */ +/* #undef LOG1P_DROPS_ZERO_SIGN */ /* Define to 1 if `major', `minor', and `makedev' are declared in . */ -#undef MAJOR_IN_MKDEV +#undef MAJOR_IN_MKDEV // original: /* #undef MAJOR_IN_MKDEV */ /* Define to 1 if `major', `minor', and `makedev' are declared in . */ @@ -983,25 +1234,20 @@ /* #undef PYLONG_BITS_IN_DIGIT */ /* Define to printf format modifier for long long type */ -#define PY_FORMAT_LONG_LONG "ll" /* #undef PY_FORMAT_LONG_LONG */ +#define PY_FORMAT_LONG_LONG "ll" // original: #undef PY_FORMAT_LONG_LONG /* Define to printf format modifier for Py_ssize_t */ #define PY_FORMAT_SIZE_T "z" -/* Define as the integral type used for Unicode representation. */ -#define PY_UNICODE_TYPE unsigned short - /* Define if you want to build an interpreter with many run-time checks. */ /* #undef Py_DEBUG */ /* Defined if Python is built as a shared library. */ /* #undef Py_ENABLE_SHARED */ -/* Define as the size of the unicode type. */ -#define Py_UNICODE_SIZE 2 - -/* Define if you want to have a Unicode type. */ -#define Py_USING_UNICODE 1 +/* Define hash algorithm for str, bytes and memoryview. SipHash24: 1, FNV: 2, + externally defined: 0 */ +/* #undef Py_HASH_ALGORITHM */ /* assume C89 semantics that RETSIGTYPE is always void */ #define RETSIGTYPE void @@ -1009,9 +1255,6 @@ /* Define if setpgrp() must be called as setpgrp(0, 0). */ /* #undef SETPGRP_HAVE_ARG */ -/* Define this to be extension of shared libraries (including the dot!). */ -#define SHLIB_EXT ".so" - /* Define if i>>j for signed int i does not extend the sign bit when i < 0 */ /* #undef SIGNED_RIGHT_SHIFT_ZERO_FILLS */ @@ -1076,6 +1319,9 @@ /* Define if tanh(-0.) is -0., or if platform doesn't have signed zeros */ /* #undef TANH_PRESERVES_ZERO_SIGN */ +/* Library needed by timemodule.c: librt may be needed for clock_gettime() */ +/* #undef TIMEMODULE_LIB */ + /* Define to 1 if you can safely include both and . */ #define TIME_WITH_SYS_TIME 1 @@ -1085,6 +1331,9 @@ /* Define if you want to use computed gotos in ceval.c. */ /* #undef USE_COMPUTED_GOTOS */ +/* Define to use the C99 inline keyword. */ +#define USE_INLINE 1 + /* Enable extensions on AIX 3, Interix. */ #ifndef _ALL_SOURCE # define _ALL_SOURCE 1 @@ -1107,19 +1356,12 @@ #endif -/* Define if you want to use MacPython modules on MacOSX in unix-Python. */ -/* #undef USE_TOOLBOX_OBJECT_GLUE */ - /* Define if a va_list is an array of some kind */ /* #undef VA_LIST_IS_ARRAY */ /* Define if you want SIGFPE handled (see Include/pyfpe.h). */ /* #undef WANT_SIGFPE_HANDLER */ -/* Define if you want wctype.h functions to be used instead of the one - supplied by Python itself. (see Include/unicodectype.h). */ -/* #undef WANT_WCTYPE_FUNCTIONS */ - /* Define if WINDOW in curses.h offers a field _flags. */ /* #undef WINDOW_HAS_FLAGS */ @@ -1180,6 +1422,9 @@ /* Define on Linux to activate all library features */ #define _GNU_SOURCE 1 +/* Define to include mbstate_t for mbrtowc */ +/* #undef _INCLUDE__STDC_A1_SOURCE */ + /* This must be defined on some systems to enable large file support. */ /* #undef _LARGEFILE_SOURCE */ @@ -1192,15 +1437,12 @@ /* Define on NetBSD to activate all library features */ #define _NETBSD_SOURCE 1 -/* Define _OSF_SOURCE to get the makedev macro. */ -/* #undef _OSF_SOURCE */ - /* Define to 2 if the system does not provide POSIX.1 features except with this defined. */ /* #undef _POSIX_1_SOURCE */ -/* Define to activate features from IEEE Stds 1003.1-2001 */ -#define _POSIX_C_SOURCE 200112L +/* Define to activate features from IEEE Stds 1003.1-2008 */ +#define _POSIX_C_SOURCE 200809L /* Define to 1 if you need to in order for `stat' and other things to work. */ /* #undef _POSIX_SOURCE */ @@ -1222,7 +1464,7 @@ /* #undef _UINT64_T */ /* Define to the level of X/Open that your system supports */ -#define _XOPEN_SOURCE 600 +#define _XOPEN_SOURCE 700 /* Define to activate Unix95-and-earlier features */ #define _XOPEN_SOURCE_EXTENDED 1 @@ -1235,9 +1477,6 @@ /* # undef __CHAR_UNSIGNED__ */ #endif -/* Defined on Solaris to see additional function prototypes. */ -#define __EXTENSIONS__ 1 - /* Define to 'long' if doesn't define. */ /* #undef clock_t */ @@ -1247,6 +1486,12 @@ /* Define to `int' if doesn't define. */ /* #undef gid_t */ +/* Define to `__inline__' or `__inline' if that's what the C compiler + calls it, or to nothing if 'inline' is not supported under any name. */ +#ifndef __cplusplus +/* #undef inline */ +#endif + /* Define to the type of a signed integer type of width exactly 32 bits if such a type exists and the standard includes do not define it. */ /* #undef int32_t */ @@ -1271,7 +1516,7 @@ /* #undef size_t */ /* Define to `int' if does not define. */ -/* #define socklen_t int */ +#define socklen_t int // original: /* #undef socklen_t */ /* Define to `int' if doesn't define. */ /* #undef uid_t */ From 55fd5530d58f05a903c33a9326c1caf53ea3b682 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Thu, 15 Mar 2018 20:53:47 +0100 Subject: [PATCH 02/72] Fixed a few things --- python_build/Makefile | 4 ++-- python_config/pyconfig.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index dea69bd..ec97db6 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -87,7 +87,7 @@ soospatchPY: compilePY touch soospatchPY compilePY: extractedPY patchPY - cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-doc-strings --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY + cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-doc-strings --without-signal-module --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY patchPY: cp $(PYDIR)/configure $(PYDIR)/configure_old @@ -96,7 +96,7 @@ patchPY: echo ac_cv_file__dev_ptc=no >>$(PYDIR)/config.site echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old - cat $(PYDIR)/Modules/Setup.dist_old | sed 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | sed 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' >$(PYDIR)/Modules/Setup.dist + cat $(PYDIR)/Modules/Setup.dist_old | sed 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | sed 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | sed "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" >$(PYDIR)/Modules/Setup.dist touch patchPY diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index 97bb9f3..fba97fd 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -1159,10 +1159,10 @@ /* #define HAVE_WAIT4 1 */ // original: #define HAVE_WAIT4 1 /* Define to 1 if you have the `waitid' function. */ -#define HAVE_WAITID 1 +/* #define HAVE_WAITID 1 */ // original: #define HAVE_WAITID 1 /* Define to 1 if you have the `waitpid' function. */ -/* #define HAVE_WAITPID 1 */ // original: #define HAVE_WAITPID 1 +#define HAVE_WAITPID 1 /* Define if the compiler provides a wchar.h header file. */ #define HAVE_WCHAR_H 1 From 9c872d3314f160e1610729d742ca9f8670d1dd1e Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Thu, 15 Mar 2018 22:08:35 +0000 Subject: [PATCH 03/72] Update to python3 print syntax --- examples/hello.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/hello.py b/examples/hello.py index 8353051..8c71d59 100644 --- a/examples/hello.py +++ b/examples/hello.py @@ -1,4 +1,4 @@ -print """ +print(""" _ _ _ | | | | | | | | | | ___| | ___ ___ _ __ ___ ___ @@ -22,4 +22,4 @@ __/ | |___/ -Replace this file (main.py) with your own to get started. """ +Replace this file (main.py) with your own to get started. """) From a0ba0f83ace3e29b91d16926f1501cae8177fe20 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Thu, 15 Mar 2018 23:19:20 +0000 Subject: [PATCH 04/72] Update to python3 C API --- source/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/main.c b/source/main.c index 4a67d07..3a346f9 100644 --- a/source/main.c +++ b/source/main.c @@ -33,7 +33,7 @@ int main(int argc, char *argv[]) /* set up import path */ PyObject *sysPath = PySys_GetObject("path"); - PyObject *path = PyString_FromString("./"); + PyObject *path = PyUnicode_FromString("./"); PyList_Insert(sysPath, 0, path); FILE * mainpy = fopen(MAINPY, "r"); From d1b01bc1a2e1ee0a36f66e9dbb41a952c96d0d24 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Thu, 15 Mar 2018 23:44:04 +0000 Subject: [PATCH 05/72] Disable more unsupported features --- python_config/pyconfig.h | 73 +++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index fba97fd..ec18a91 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -39,7 +39,7 @@ /* #undef GETTIMEOFDAY_NO_TZ */ /* Define to 1 if you have the `accept4' function. */ -#define HAVE_ACCEPT4 1 +/* #define HAVE_ACCEPT4 1 */ // original: #define HAVE_ACCEPT4 1 /* Define to 1 if you have the `acosh' function. */ #define HAVE_ACOSH 1 @@ -121,10 +121,10 @@ #define HAVE_CLOCK 1 /* Define to 1 if you have the `clock_getres' function. */ -#define HAVE_CLOCK_GETRES 1 +/* #define HAVE_CLOCK_GETRES 1 */ // original: #define HAVE_CLOCK_GETRES 1 /* Define to 1 if you have the `clock_gettime' function. */ -#define HAVE_CLOCK_GETTIME 1 +/* #define HAVE_CLOCK_GETTIME 1 */ // original: #define HAVE_CLOCK_GETTIME 1 /* Define if the C compiler supports computed gotos. */ /* #undef HAVE_COMPUTED_GOTOS */ @@ -206,6 +206,9 @@ /* Define to 1 if you have the `dup3' function. */ #define HAVE_DUP3 1 +/* O_CLOEXEC is not supported, but ignoring it should be fine here */ +#define O_CLOEXEC 0 + /* Defined when any dynamic module loading is enabled. */ #undef HAVE_DYNAMIC_LOADING // original: #define HAVE_DYNAMIC_LOADING 1 @@ -234,7 +237,7 @@ #define HAVE_EXPM1 1 /* Define to 1 if you have the `faccessat' function. */ -#define HAVE_FACCESSAT 1 +/* #define HAVE_FACCESSAT 1 */ // original: #define HAVE_FACCESSAT 1 /* Define if you have the 'fchdir' function. */ /* #define HAVE_FCHDIR 1 */ // #define HAVE_FCHDIR 1 @@ -246,7 +249,7 @@ #define HAVE_FCHMODAT 1 /* Define to 1 if you have the `fchown' function. */ -#define HAVE_FCHOWN 1 +/* #define HAVE_FCHOWN 1 */ // original: #define HAVE_FCHOWN 1 /* Define to 1 if you have the `fchownat' function. */ #define HAVE_FCHOWNAT 1 @@ -258,7 +261,7 @@ #undef HAVE_FDATASYNC // original: #define HAVE_FDATASYNC 1 /* Define to 1 if you have the `fdopendir' function. */ -#define HAVE_FDOPENDIR 1 +/* #define HAVE_FDOPENDIR 1 */ // original: #define HAVE_FDOPENDIR 1 /* Define to 1 if you have the `fexecve' function. */ #define HAVE_FEXECVE 1 @@ -285,7 +288,7 @@ #define HAVE_FSEEKO 1 /* Define to 1 if you have the `fstatat' function. */ -#define HAVE_FSTATAT 1 +/* #define HAVE_FSTATAT 1 */ // original: #define HAVE_FSTATAT 1 /* Define to 1 if you have the `fstatvfs' function. */ #define HAVE_FSTATVFS 1 @@ -306,7 +309,7 @@ #define HAVE_FTRUNCATE 1 /* Define to 1 if you have the `futimens' function. */ -#define HAVE_FUTIMENS 1 +/* #define HAVE_FUTIMENS 1 */ // original: #define HAVE_FUTIMENS 1 /* Define to 1 if you have the `futimes' function. */ #define HAVE_FUTIMES 1 @@ -350,7 +353,7 @@ #undef HAVE_GETENTROPY // original: #define HAVE_GETENTROPY 1 /* Define to 1 if you have the `getgrouplist' function. */ -#define HAVE_GETGROUPLIST 1 +/* #define HAVE_GETGROUPLIST 1 */ // original: #define HAVE_GETGROUPLIST 1 /* Define to 1 if you have the `getgroups' function. */ /* #define HAVE_GETGROUPS 1 */ // original: #define HAVE_GETGROUPS 1 @@ -398,7 +401,7 @@ #define HAVE_GETPID 1 /* Define to 1 if you have the `getpriority' function. */ -#define HAVE_GETPRIORITY 1 +/* #define HAVE_GETPRIORITY 1 */ // original: #define HAVE_GETPRIORITY 1 /* Define to 1 if you have the `getpwent' function. */ #undef HAVE_GETPWENT // original: #define HAVE_GETPWENT 1 @@ -450,7 +453,7 @@ #define HAVE_IEEEFP_H 1 /* Define to 1 if you have the `if_nameindex' function. */ -#define HAVE_IF_NAMEINDEX 1 +/* #define HAVE_IF_NAMEINDEX 1 */ // original: #define HAVE_IF_NAMEINDEX 1 /* Define if you have the 'inet_aton' function. */ /* #define HAVE_INET_ATON 1 */ // original: #define HAVE_INET_ATON 1 @@ -534,7 +537,7 @@ #define HAVE_LINK 1 /* Define to 1 if you have the `linkat' function. */ -#define HAVE_LINKAT 1 +/* #define HAVE_LINKAT 1 */ // original: #define HAVE_LINKAT 1 /* Define to 1 if you have the header file. */ /* #undef HAVE_LINUX_CAN_BCM_H */ @@ -558,7 +561,7 @@ /* #undef HAVE_LINUX_TIPC_H */ /* Define to 1 if you have the `lockf' function. */ -#define HAVE_LOCKF 1 +/* #define HAVE_LOCKF 1 */ // original: #define HAVE_LOCKF 1 /* Define to 1 if you have the `log1p' function. */ #define HAVE_LOG1P 1 @@ -594,7 +597,7 @@ #define HAVE_MEMRCHR 1 /* Define to 1 if you have the `mkdirat' function. */ -#define HAVE_MKDIRAT 1 +/* #define HAVE_MKDIRAT 1 */ // #define HAVE_MKDIRAT 1 /* Define to 1 if you have the `mkfifo' function. */ /* #define HAVE_MKFIFO 1 */ // original: #define HAVE_MKFIFO 1 @@ -633,7 +636,7 @@ /* #define HAVE_NICE 1 */ // original: #define HAVE_NICE 1 /* Define to 1 if you have the `openat' function. */ -#define HAVE_OPENAT 1 +/* #define HAVE_OPENAT 1 */ // original: #define HAVE_OPENAT 1 /* Define to 1 if you have the `openpty' function. */ #undef HAVE_OPENPTY // original: #define HAVE_OPENPTY 1 @@ -660,13 +663,13 @@ /* #undef HAVE_POLL_H */ // original: #define HAVE_POLL_H 1 /* Define to 1 if you have the `posix_fadvise' function. */ -#define HAVE_POSIX_FADVISE 1 +/* #define HAVE_POSIX_FADVISE 1 */ // original: #define HAVE_POSIX_FADVISE 1 /* Define to 1 if you have the `posix_fallocate' function. */ -#define HAVE_POSIX_FALLOCATE 1 +/* #define HAVE_POSIX_FALLOCATE 1 */ // original: #define HAVE_POSIX_FALLOCATE 1 /* Define to 1 if you have the `pread' function. */ -#define HAVE_PREAD 1 +/* #define HAVE_PREAD 1 */ // original: #define HAVE_PREAD 1 /* Define if you have the 'prlimit' functions. */ /* #undef HAVE_PRLIMIT */ @@ -702,7 +705,7 @@ #define HAVE_PUTENV 1 /* Define to 1 if you have the `pwrite' function. */ -#define HAVE_PWRITE 1 +/* #define HAVE_PWRITE 1 */ // original: #define HAVE_PWRITE 1 /* Define to 1 if you have the `readlink' function. */ #undef HAVE_READLINK // original: #define HAVE_READLINK 1 @@ -711,13 +714,13 @@ #define HAVE_READLINKAT 1 /* Define to 1 if you have the `readv' function. */ -#define HAVE_READV 1 +/* #define HAVE_READV 1 */ // original: #define HAVE_READV 1 /* Define to 1 if you have the `realpath' function. */ #define HAVE_REALPATH 1 /* Define to 1 if you have the `renameat' function. */ -#define HAVE_RENAMEAT 1 +/* #define HAVE_RENAMEAT 1 */ // original: #define HAVE_RENAMEAT 1 /* Define if readline supports append_history */ /* #undef HAVE_RL_APPEND_HISTORY */ @@ -750,22 +753,22 @@ #define HAVE_ROUND 1 /* Define to 1 if you have the `sched_get_priority_max' function. */ -#define HAVE_SCHED_GET_PRIORITY_MAX 1 +/* #define HAVE_SCHED_GET_PRIORITY_MAX 1 */ // original: #define HAVE_SCHED_GET_PRIORITY_MAX 1 /* Define to 1 if you have the header file. */ #define HAVE_SCHED_H 1 /* Define to 1 if you have the `sched_rr_get_interval' function. */ -#define HAVE_SCHED_RR_GET_INTERVAL 1 +/* #define HAVE_SCHED_RR_GET_INTERVAL 1 */ // original: #define HAVE_SCHED_RR_GET_INTERVAL 1 /* Define to 1 if you have the `sched_setaffinity' function. */ #define HAVE_SCHED_SETAFFINITY 1 /* Define to 1 if you have the `sched_setparam' function. */ -#define HAVE_SCHED_SETPARAM 1 +/* #define HAVE_SCHED_SETPARAM 1 */ //original: #define HAVE_SCHED_SETPARAM 1 /* Define to 1 if you have the `sched_setscheduler' function. */ -#define HAVE_SCHED_SETSCHEDULER 1 +/* #define HAVE_SCHED_SETSCHEDULER 1 */ // original: #define HAVE_SCHED_SETSCHEDULER 1 /* Define to 1 if you have the `select' function. */ #define HAVE_SELECT 1 @@ -783,7 +786,7 @@ #define HAVE_SEM_UNLINK 1 /* Define to 1 if you have the `sendfile' function. */ -#define HAVE_SENDFILE 1 +/* #define HAVE_SENDFILE 1 */ // original: #define HAVE_SENDFILE 1 /* Define to 1 if you have the `setegid' function. */ /* #define HAVE_SETEGID 1 */ // original: #define HAVE_SETEGID 1 @@ -798,7 +801,7 @@ /* #define HAVE_SETGROUPS 1 */ // original: #define HAVE_SETGROUPS 1 /* Define to 1 if you have the `sethostname' function. */ -#define HAVE_SETHOSTNAME 1 +/* #define HAVE_SETHOSTNAME 1 */ // original: #define HAVE_SETHOSTNAME 1 /* Define to 1 if you have the `setitimer' function. */ /* #define HAVE_SETITIMER 1 */ // original: #define HAVE_SETITIMER 1 @@ -813,7 +816,7 @@ /* #define HAVE_SETPGRP 1 */ // original: #define HAVE_SETPGRP 1 /* Define to 1 if you have the `setpriority' function. */ -#define HAVE_SETPRIORITY 1 +/* #define HAVE_SETPRIORITY 1 */ // original: #define HAVE_SETPRIORITY 1 /* Define to 1 if you have the `setregid' function. */ /* #define HAVE_SETREGID 1 */ // original: #define HAVE_SETREGID 1 @@ -843,7 +846,7 @@ #undef HAVE_SIGACTION // original: #define HAVE_SIGACTION 1 /* Define to 1 if you have the `sigaltstack' function. */ -#define HAVE_SIGALTSTACK 1 +/* #define HAVE_SIGALTSTACK 1 */ // original: #define HAVE_SIGALTSTACK 1 /* Define to 1 if you have the `siginterrupt' function. */ #undef HAVE_SIGINTERRUPT // original: #define HAVE_SIGINTERRUPT 1 @@ -952,7 +955,7 @@ #define HAVE_SYMLINKAT 1 /* Define to 1 if you have the `sync' function. */ -#define HAVE_SYNC 1 +/* #define HAVE_SYNC 1 */ // original: #define HAVE_SYNC 1 /* Define to 1 if you have the `sysconf' function. */ /* #define HAVE_SYSCONF 1 */ // original: #define HAVE_SYSCONF 1 @@ -1017,7 +1020,7 @@ /* #undef HAVE_SYS_RANDOM_H */ /* Define to 1 if you have the header file. */ -#define HAVE_SYS_RESOURCE_H 1 +/* #define HAVE_SYS_RESOURCE_H 1 */ // original: #define HAVE_SYS_RESOURCE_H 1 /* Define to 1 if you have the header file. */ #define HAVE_SYS_SELECT_H 1 @@ -1130,7 +1133,7 @@ #define HAVE_UNISTD_H 1 /* Define to 1 if you have the `unlinkat' function. */ -#define HAVE_UNLINKAT 1 +/* #define HAVE_UNLINKAT 1 */ // original: #define HAVE_UNLINKAT 1 /* Define to 1 if you have the `unsetenv' function. */ #define HAVE_UNSETENV 1 @@ -1144,7 +1147,7 @@ /* #undef HAVE_UTIL_H */ /* Define to 1 if you have the `utimensat' function. */ -#define HAVE_UTIMENSAT 1 +/* #define HAVE_UTIMENSAT 1 */ // original: #define HAVE_UTIMENSAT 1 /* Define to 1 if you have the `utimes' function. */ #define HAVE_UTIMES 1 @@ -1162,7 +1165,7 @@ /* #define HAVE_WAITID 1 */ // original: #define HAVE_WAITID 1 /* Define to 1 if you have the `waitpid' function. */ -#define HAVE_WAITPID 1 +/* #define HAVE_WAITPID 1 */ // original: #define HAVE_WAITPID 1 /* Define if the compiler provides a wchar.h header file. */ #define HAVE_WCHAR_H 1 @@ -1516,7 +1519,7 @@ /* #undef size_t */ /* Define to `int' if does not define. */ -#define socklen_t int // original: /* #undef socklen_t */ +/* #define socklen_t int */ /* Define to `int' if doesn't define. */ /* #undef uid_t */ From 82a8eb9effe04eb2891a792dc272d285e1ce3afc Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Thu, 15 Mar 2018 23:48:54 +0000 Subject: [PATCH 06/72] Use wchar with Py_SetProgramName --- source/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/main.c b/source/main.c index 3a346f9..58bdeeb 100644 --- a/source/main.c +++ b/source/main.c @@ -24,7 +24,7 @@ int main(int argc, char *argv[]) //Py_VerboseFlag += 1; /* not the real path, but this should make it look for modules in ./lib */ - Py_SetProgramName("./bin/python"); + Py_SetProgramName(L"./bin/python"); Py_Initialize(); From b5993dee7e25ca372912142f08b2562c2ebc98ef Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Thu, 15 Mar 2018 23:54:20 +0000 Subject: [PATCH 07/72] More pyconfig fixes --- python_config/pyconfig.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index ec18a91..fe67dfe 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -204,10 +204,7 @@ #define HAVE_DUP2 1 /* Define to 1 if you have the `dup3' function. */ -#define HAVE_DUP3 1 - -/* O_CLOEXEC is not supported, but ignoring it should be fine here */ -#define O_CLOEXEC 0 +/* #define HAVE_DUP3 1 */ // original: #define HAVE_DUP3 1 /* Defined when any dynamic module loading is enabled. */ #undef HAVE_DYNAMIC_LOADING // original: #define HAVE_DYNAMIC_LOADING 1 @@ -246,7 +243,7 @@ #define HAVE_FCHMOD 1 /* Define to 1 if you have the `fchmodat' function. */ -#define HAVE_FCHMODAT 1 +/* #define HAVE_FCHMODAT 1 */ // original: #define HAVE_FCHMODAT 1 /* Define to 1 if you have the `fchown' function. */ /* #define HAVE_FCHOWN 1 */ // original: #define HAVE_FCHOWN 1 @@ -312,10 +309,10 @@ /* #define HAVE_FUTIMENS 1 */ // original: #define HAVE_FUTIMENS 1 /* Define to 1 if you have the `futimes' function. */ -#define HAVE_FUTIMES 1 +/* #define HAVE_FUTIMES 1 */ // original: #define HAVE_FUTIMES 1 /* Define to 1 if you have the `futimesat' function. */ -#define HAVE_FUTIMESAT 1 +/* #define HAVE_FUTIMESAT 1 */ // original: #define HAVE_FUTIMESAT 1 /* Define to 1 if you have the `gai_strerror' function. */ #define HAVE_GAI_STRERROR 1 @@ -579,7 +576,7 @@ /* #define HAVE_LSTAT 1 */ // original: #define HAVE_LSTAT 1 /* Define to 1 if you have the `lutimes' function. */ -#define HAVE_LUTIMES 1 +/* #define HAVE_LUTIMES 1 */ // original: #define HAVE_LUTIMES 1 /* Define this if you have the makedev macro. */ /* #define HAVE_MAKEDEV 1 */ // original: #define HAVE_MAKEDEV 1 @@ -651,7 +648,7 @@ #undef HAVE_PAUSE // original: #define HAVE_PAUSE 1 /* Define to 1 if you have the `pipe2' function. */ -#define HAVE_PIPE2 1 +/* #define HAVE_PIPE2 1 */ // original: #define HAVE_PIPE2 1 /* Define to 1 if you have the `plock' function. */ /* #define HAVE_PLOCK 1 */ // original: #define HAVE_PLOCK 1 @@ -1187,7 +1184,7 @@ /* #undef HAVE_WORKING_TZSET */ /* Define to 1 if you have the `writev' function. */ -#define HAVE_WRITEV 1 +/* #define HAVE_WRITEV 1 */ // orignal: #define HAVE_WRITEV 1 /* Define if the zlib library has inflateCopy */ /* #undef HAVE_ZLIB_COPY */ From 8d60a7e22106b7fcad2e9f20776036391c4b9468 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Fri, 16 Mar 2018 00:07:47 +0000 Subject: [PATCH 08/72] Add some source patches to makefile --- python_build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index ec97db6..6cb2f15 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -75,7 +75,7 @@ linkPY: soospatchPY soospatchPY: compilePY cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/pyconfig.h cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old - cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/access(path, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*i = (int)umask(i);.*\)/return NULL; \1/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c + cat $(PYDIR)/Modules/posixmodule.c_old | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*i = (int)umask(i);.*\)/return NULL; \1/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old cat $(PYDIR)/Modules/socketmodule.c_old | sed 's/send(s->sock_fd, buf, len, flags);/send(s->sock_fd, buf, len<4096?len:4096, flags);/g' | sed 's/ sizeof(addr->sa_data)/ 28/g' >$(PYDIR)/Modules/socketmodule.c cp $(PYDIR)/Objects/exceptions.c $(PYDIR)/Objects/exceptions.c_old @@ -83,7 +83,7 @@ soospatchPY: compilePY cp $(PYDIR)/Python/pytime.c $(PYDIR)/Python/pytime.c_old cat $(PYDIR)/Python/pytime.c_old | sed 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c cp $(PYDIR)/Makefile $(PYDIR)/Makefile_old - cat $(PYDIR)/Makefile_old | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile + cat $(PYDIR)/Makefile_old | sed 's/Parser\/intrcheck.o//' | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile touch soospatchPY compilePY: extractedPY patchPY From 5f13b69dad4d01201f7de982b8fbe8b194d67d50 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Fri, 16 Mar 2018 00:33:27 +0000 Subject: [PATCH 09/72] re-add intrcheck.c, un-break posixmodule patch --- python_build/Makefile | 7 +- python_config/intrcheck.c | 180 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 python_config/intrcheck.c diff --git a/python_build/Makefile b/python_build/Makefile index 6cb2f15..2b141a8 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -73,9 +73,10 @@ linkPY: soospatchPY cd $(PYDIR) && make $(MAKEFLAGS) LIBRARY="$(ANAME)" LDLIBRARY="$(ANAME)" $(ANAME) && cd .. && cp $(PYDIR)/$(ANAME) . && touch linkPY soospatchPY: compilePY - cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/pyconfig.h + cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ + cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old - cat $(PYDIR)/Modules/posixmodule.c_old | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*i = (int)umask(i);.*\)/return NULL; \1/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c + cat $(PYDIR)/Modules/posixmodule.c_old | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*i = (int)umask(i);.*\)/return NULL; \1/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old cat $(PYDIR)/Modules/socketmodule.c_old | sed 's/send(s->sock_fd, buf, len, flags);/send(s->sock_fd, buf, len<4096?len:4096, flags);/g' | sed 's/ sizeof(addr->sa_data)/ 28/g' >$(PYDIR)/Modules/socketmodule.c cp $(PYDIR)/Objects/exceptions.c $(PYDIR)/Objects/exceptions.c_old @@ -83,7 +84,7 @@ soospatchPY: compilePY cp $(PYDIR)/Python/pytime.c $(PYDIR)/Python/pytime.c_old cat $(PYDIR)/Python/pytime.c_old | sed 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c cp $(PYDIR)/Makefile $(PYDIR)/Makefile_old - cat $(PYDIR)/Makefile_old | sed 's/Parser\/intrcheck.o//' | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile + cat $(PYDIR)/Makefile_old | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile touch soospatchPY compilePY: extractedPY patchPY diff --git a/python_config/intrcheck.c b/python_config/intrcheck.c new file mode 100644 index 0000000..67df007 --- /dev/null +++ b/python_config/intrcheck.c @@ -0,0 +1,180 @@ +/* This file was removed from newer python versions, so it was +rescued from a python 2.7 branch... */ + +/* Check for interrupts */ + +#include "Python.h" +#include "pythread.h" + +#ifdef QUICKWIN + +#include + +void +PyOS_InitInterrupts(void) +{ +} + +void +PyOS_FiniInterrupts(void) +{ +} + +int +PyOS_InterruptOccurred(void) +{ + _wyield(); +} + +#define OK + +#endif /* QUICKWIN */ + +#if defined(_M_IX86) && !defined(__QNX__) +#include +#endif + +#if defined(MSDOS) && !defined(QUICKWIN) + +#ifdef __GNUC__ + +/* This is for DJGPP's GO32 extender. I don't know how to trap + * control-C (There's no API for ctrl-C, and I don't want to mess with + * the interrupt vectors.) However, this DOES catch control-break. + * --Amrit + */ + +#include + +void +PyOS_InitInterrupts(void) +{ + _go32_want_ctrl_break(1 /* TRUE */); +} + +void +PyOS_FiniInterrupts(void) +{ +} + +int +PyOS_InterruptOccurred(void) +{ + return _go32_was_ctrl_break_hit(); +} + +#else /* !__GNUC__ */ + +/* This might work for MS-DOS (untested though): */ + +void +PyOS_InitInterrupts(void) +{ +} + +void +PyOS_FiniInterrupts(void) +{ +} + +int +PyOS_InterruptOccurred(void) +{ + int interrupted = 0; + while (kbhit()) { + if (getch() == '\003') + interrupted = 1; + } + return interrupted; +} + +#endif /* __GNUC__ */ + +#define OK + +#endif /* MSDOS && !QUICKWIN */ + + +#ifndef OK + +/* Default version -- for real operating systems and for Standard C */ + +#include +#include +#include + +static int interrupted; + +void +PyErr_SetInterrupt(void) +{ + interrupted = 1; +} + +extern int PyErr_CheckSignals(void); + +static int +checksignals_witharg(void * arg) +{ + return PyErr_CheckSignals(); +} + +static void +intcatcher(int sig) +{ + extern void Py_Exit(int); + static char message[] = +"python: to interrupt a truly hanging Python program, interrupt once more.\n"; + switch (interrupted++) { + case 0: + break; + case 1: +#ifdef RISCOS + fprintf(stderr, message); +#else + write(2, message, strlen(message)); +#endif + break; + case 2: + interrupted = 0; + Py_Exit(1); + break; + } + PyOS_setsig(SIGINT, intcatcher); + Py_AddPendingCall(checksignals_witharg, NULL); +} + +static void (*old_siginthandler)(int) = SIG_DFL; + +void +PyOS_InitInterrupts(void) +{ + if ((old_siginthandler = PyOS_setsig(SIGINT, SIG_IGN)) != SIG_IGN) + PyOS_setsig(SIGINT, intcatcher); +} + +void +PyOS_FiniInterrupts(void) +{ + PyOS_setsig(SIGINT, old_siginthandler); +} + +int +PyOS_InterruptOccurred(void) +{ + if (!interrupted) + return 0; + interrupted = 0; + return 1; +} + +#endif /* !OK */ + +void +PyOS_AfterFork(void) +{ +#ifdef WITH_THREAD + PyEval_ReInitThreads(); + PyThread_ReInitTLS(); +#endif +} From 1462790591b68e8ebd27d2b4f3d6db8f3039d24d Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Fri, 16 Mar 2018 00:45:44 +0000 Subject: [PATCH 10/72] More config fixes --- python_config/pyconfig.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index fe67dfe..41a9fc9 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -753,7 +753,7 @@ /* #define HAVE_SCHED_GET_PRIORITY_MAX 1 */ // original: #define HAVE_SCHED_GET_PRIORITY_MAX 1 /* Define to 1 if you have the header file. */ -#define HAVE_SCHED_H 1 +/* #define HAVE_SCHED_H 1 */ // original: #define HAVE_SCHED_H 1 /* Define to 1 if you have the `sched_rr_get_interval' function. */ /* #define HAVE_SCHED_RR_GET_INTERVAL 1 */ // original: #define HAVE_SCHED_RR_GET_INTERVAL 1 @@ -1147,7 +1147,7 @@ /* #define HAVE_UTIMENSAT 1 */ // original: #define HAVE_UTIMENSAT 1 /* Define to 1 if you have the `utimes' function. */ -#define HAVE_UTIMES 1 +/* #define HAVE_UTIMES 1 */ // original: #define HAVE_UTIMES 1 /* Define to 1 if you have the header file. */ /* #define HAVE_UTIME_H 1 */ // original: #define HAVE_UTIME_H 1 From b94a0c18d9100ae340eb66d4fde51e727e92edb8 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Fri, 16 Mar 2018 01:15:06 +0000 Subject: [PATCH 11/72] Patch out utime, lstat and umask --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 2b141a8..e9e2e76 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -76,7 +76,7 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old - cat $(PYDIR)/Modules/posixmodule.c_old | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*i = (int)umask(i);.*\)/return NULL; \1/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c + cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old cat $(PYDIR)/Modules/socketmodule.c_old | sed 's/send(s->sock_fd, buf, len, flags);/send(s->sock_fd, buf, len<4096?len:4096, flags);/g' | sed 's/ sizeof(addr->sa_data)/ 28/g' >$(PYDIR)/Modules/socketmodule.c cp $(PYDIR)/Objects/exceptions.c $(PYDIR)/Objects/exceptions.c_old From f6de68864a5b24118a4f6b6c35767bf9d164bfea Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Fri, 16 Mar 2018 08:09:15 +0000 Subject: [PATCH 12/72] Explicitly set Python Home --- source/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/main.c b/source/main.c index 58bdeeb..99f948a 100644 --- a/source/main.c +++ b/source/main.c @@ -23,8 +23,7 @@ int main(int argc, char *argv[]) Py_NoUserSiteDirectory = 1; //Py_VerboseFlag += 1; - /* not the real path, but this should make it look for modules in ./lib */ - Py_SetProgramName(L"./bin/python"); + Py_SetPythonHome(L"./"); Py_Initialize(); From 2b646082f1fb466b1878faf5fec3410977c29007 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 00:08:57 +0000 Subject: [PATCH 13/72] Use patched copy of pytime.c --- python_build/Makefile | 1 + python_config/pytime.c | 696 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 697 insertions(+) create mode 100644 python_config/pytime.c diff --git a/python_build/Makefile b/python_build/Makefile index e9e2e76..652e83a 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -75,6 +75,7 @@ linkPY: soospatchPY soospatchPY: compilePY cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ + cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old diff --git a/python_config/pytime.c b/python_config/pytime.c new file mode 100644 index 0000000..37fa5cd --- /dev/null +++ b/python_config/pytime.c @@ -0,0 +1,696 @@ +#include "Python.h" +#ifdef MS_WINDOWS +#include +#endif + +#if defined(__APPLE__) +#include /* mach_absolute_time(), mach_timebase_info() */ +#endif + +/* To millisecond (10^-3) */ +#define SEC_TO_MS 1000 + +/* To microseconds (10^-6) */ +#define MS_TO_US 1000 +#define SEC_TO_US (SEC_TO_MS * MS_TO_US) + +/* To nanoseconds (10^-9) */ +#define US_TO_NS 1000 +#define MS_TO_NS (MS_TO_US * US_TO_NS) +#define SEC_TO_NS (SEC_TO_MS * MS_TO_NS) + +/* Conversion from nanoseconds */ +#define NS_TO_MS (1000 * 1000) +#define NS_TO_US (1000) + +static void +error_time_t_overflow(void) +{ + PyErr_SetString(PyExc_OverflowError, + "timestamp out of range for platform time_t"); +} + +time_t +_PyLong_AsTime_t(PyObject *obj) +{ +#if defined(HAVE_LONG_LONG) && SIZEOF_TIME_T == SIZEOF_LONG_LONG + PY_LONG_LONG val; + val = PyLong_AsLongLong(obj); +#else + long val; + assert(sizeof(time_t) <= sizeof(long)); + val = PyLong_AsLong(obj); +#endif + if (val == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) + error_time_t_overflow(); + return -1; + } + return (time_t)val; +} + +PyObject * +_PyLong_FromTime_t(time_t t) +{ +#if defined(HAVE_LONG_LONG) && SIZEOF_TIME_T == SIZEOF_LONG_LONG + return PyLong_FromLongLong((PY_LONG_LONG)t); +#else + assert(sizeof(time_t) <= sizeof(long)); + return PyLong_FromLong((long)t); +#endif +} + +static int +_PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, + double denominator, _PyTime_round_t round) +{ + assert(denominator <= LONG_MAX); + if (PyFloat_Check(obj)) { + double d, intpart, err; + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double floatpart; + + d = PyFloat_AsDouble(obj); + floatpart = modf(d, &intpart); + if (floatpart < 0) { + floatpart = 1.0 + floatpart; + intpart -= 1.0; + } + + floatpart *= denominator; + if (round == _PyTime_ROUND_CEILING) { + floatpart = ceil(floatpart); + if (floatpart >= denominator) { + floatpart = 0.0; + intpart += 1.0; + } + } + else { + floatpart = floor(floatpart); + } + + *sec = (time_t)intpart; + err = intpart - (double)*sec; + if (err <= -1.0 || err >= 1.0) { + error_time_t_overflow(); + return -1; + } + + *numerator = (long)floatpart; + return 0; + } + else { + *sec = _PyLong_AsTime_t(obj); + if (*sec == (time_t)-1 && PyErr_Occurred()) + return -1; + *numerator = 0; + return 0; + } +} + +int +_PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) +{ + if (PyFloat_Check(obj)) { + double d, intpart, err; + + d = PyFloat_AsDouble(obj); + if (round == _PyTime_ROUND_CEILING) + d = ceil(d); + else + d = floor(d); + (void)modf(d, &intpart); + + *sec = (time_t)intpart; + err = intpart - (double)*sec; + if (err <= -1.0 || err >= 1.0) { + error_time_t_overflow(); + return -1; + } + return 0; + } + else { + *sec = _PyLong_AsTime_t(obj); + if (*sec == (time_t)-1 && PyErr_Occurred()) + return -1; + return 0; + } +} + +int +_PyTime_ObjectToTimespec(PyObject *obj, time_t *sec, long *nsec, + _PyTime_round_t round) +{ + return _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); +} + +int +_PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, + _PyTime_round_t round) +{ + return _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); +} + +static void +_PyTime_overflow(void) +{ + PyErr_SetString(PyExc_OverflowError, + "timestamp too large to convert to C _PyTime_t"); +} + +_PyTime_t +_PyTime_FromSeconds(int seconds) +{ + _PyTime_t t; + /* ensure that integer overflow cannot happen, int type should have 32 + bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_MS takes 30 + bits). */ + assert((seconds >= 0 && seconds <= _PyTime_MAX / SEC_TO_NS) + || (seconds < 0 && seconds >= _PyTime_MIN / SEC_TO_NS)); + t = (_PyTime_t)seconds * SEC_TO_NS; + return t; +} + +_PyTime_t +_PyTime_FromNanoseconds(PY_LONG_LONG ns) +{ + _PyTime_t t; + assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + t = Py_SAFE_DOWNCAST(ns, PY_LONG_LONG, _PyTime_t); + return t; +} + +#ifdef HAVE_CLOCK_GETTIME +static int +_PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) +{ + _PyTime_t t; + int res = 0; + + t = (_PyTime_t)ts->tv_sec * SEC_TO_NS; + if (t / SEC_TO_NS != ts->tv_sec) { + if (raise) + _PyTime_overflow(); + res = -1; + } + + t += ts->tv_nsec; + + *tp = t; + return res; +} +#elif !defined(MS_WINDOWS) +static int +_PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) +{ + _PyTime_t t; + int res = 0; + + t = (_PyTime_t)tv->tv_sec * SEC_TO_NS; + if (t / SEC_TO_NS != tv->tv_sec) { + if (raise) + _PyTime_overflow(); + res = -1; + } + + t += (_PyTime_t)tv->tv_usec * US_TO_NS; + + *tp = t; + return res; +} +#endif + +static int +_PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, + long to_nanoseconds) +{ + if (PyFloat_Check(obj)) { + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double d, err; + + /* convert to a number of nanoseconds */ + d = PyFloat_AsDouble(obj); + d *= to_nanoseconds; + + if (round == _PyTime_ROUND_CEILING) + d = ceil(d); + else + d = floor(d); + + *t = (_PyTime_t)d; + err = d - (double)*t; + if (fabs(err) >= 1.0) { + _PyTime_overflow(); + return -1; + } + return 0; + } + else { +#ifdef HAVE_LONG_LONG + PY_LONG_LONG sec; + sec = PyLong_AsLongLong(obj); + assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); +#else + long sec; + sec = PyLong_AsLong(obj); + assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); +#endif + if (sec == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) + _PyTime_overflow(); + return -1; + } + *t = sec * to_nanoseconds; + if (*t / to_nanoseconds != sec) { + _PyTime_overflow(); + return -1; + } + return 0; + } +} + +int +_PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) +{ + return _PyTime_FromObject(t, obj, round, SEC_TO_NS); +} + +int +_PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round) +{ + return _PyTime_FromObject(t, obj, round, MS_TO_NS); +} + +double +_PyTime_AsSecondsDouble(_PyTime_t t) +{ + _PyTime_t sec, ns; + /* Divide using integers to avoid rounding issues on the integer part. + 1e-9 cannot be stored exactly in IEEE 64-bit. */ + sec = t / SEC_TO_NS; + ns = t % SEC_TO_NS; + return (double)sec + (double)ns * 1e-9; +} + +PyObject * +_PyTime_AsNanosecondsObject(_PyTime_t t) +{ +#ifdef HAVE_LONG_LONG + assert(sizeof(PY_LONG_LONG) >= sizeof(_PyTime_t)); + return PyLong_FromLongLong((PY_LONG_LONG)t); +#else + assert(sizeof(long) >= sizeof(_PyTime_t)); + return PyLong_FromLong((long)t); +#endif +} + +static _PyTime_t +_PyTime_Divide(const _PyTime_t t, const _PyTime_t k, + const _PyTime_round_t round) +{ + assert(k > 1); + if (round == _PyTime_ROUND_CEILING) { + if (t >= 0) + return (t + k - 1) / k; + else + return t / k; + } + else { + if (t >= 0) + return t / k; + else + return (t - (k - 1)) / k; + } +} + +_PyTime_t +_PyTime_AsMilliseconds(_PyTime_t t, _PyTime_round_t round) +{ + return _PyTime_Divide(t, NS_TO_MS, round); +} + +_PyTime_t +_PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) +{ + return _PyTime_Divide(t, NS_TO_US, round); +} + +static int +_PyTime_AsTimeval_impl(_PyTime_t t, _PyTime_t *p_secs, int *p_us, + _PyTime_round_t round) +{ + _PyTime_t secs, ns; + int usec; + int res = 0; + + secs = t / SEC_TO_NS; + ns = t % SEC_TO_NS; + + usec = (int)_PyTime_Divide(ns, US_TO_NS, round); + if (usec < 0) { + usec += SEC_TO_US; + if (secs != _PyTime_MIN) + secs -= 1; + else + res = -1; + } + else if (usec >= SEC_TO_US) { + usec -= SEC_TO_US; + if (secs != _PyTime_MAX) + secs += 1; + else + res = -1; + } + assert(0 <= usec && usec < SEC_TO_US); + + *p_secs = secs; + *p_us = usec; + + return res; +} + +static int +_PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, + _PyTime_round_t round, int raise) +{ + _PyTime_t secs; + int us; + int res; + + res = _PyTime_AsTimeval_impl(t, &secs, &us, round); + +#ifdef MS_WINDOWS + tv->tv_sec = (long)secs; +#else + tv->tv_sec = secs; +#endif + tv->tv_usec = us; + + if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + if (raise) + error_time_t_overflow(); + return -1; + } + return 0; +} + +int +_PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +{ + return _PyTime_AsTimevalStruct_impl(t, tv, round, 1); +} + +int +_PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) +{ + return _PyTime_AsTimevalStruct_impl(t, tv, round, 0); +} + +int +_PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, + _PyTime_round_t round) +{ + _PyTime_t secs; + int res; + + res = _PyTime_AsTimeval_impl(t, &secs, us, round); + + *p_secs = secs; + + if (res < 0 || (_PyTime_t)*p_secs != secs) { + error_time_t_overflow(); + return -1; + } + return 0; +} + +#if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) +int +_PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) +{ + _PyTime_t secs, nsec; + + secs = t / SEC_TO_NS; + nsec = t % SEC_TO_NS; + if (nsec < 0) { + nsec += SEC_TO_NS; + secs -= 1; + } + ts->tv_sec = (time_t)secs; + if ((_PyTime_t)ts->tv_sec != secs) { + _PyTime_overflow(); + return -1; + } + ts->tv_nsec = nsec; + + assert(0 <= ts->tv_nsec && ts->tv_nsec <= 999999999); + return 0; +} +#endif + +static int +pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) +{ +#ifdef MS_WINDOWS + FILETIME system_time; + ULARGE_INTEGER large; + + assert(info == NULL || raise); + + GetSystemTimeAsFileTime(&system_time); + large.u.LowPart = system_time.dwLowDateTime; + large.u.HighPart = system_time.dwHighDateTime; + /* 11,644,473,600,000,000,000: number of nanoseconds between + the 1st january 1601 and the 1st january 1970 (369 years + 89 leap + days). */ + *tp = large.QuadPart * 100 - 11644473600000000000; + if (info) { + DWORD timeAdjustment, timeIncrement; + BOOL isTimeAdjustmentDisabled, ok; + + info->implementation = "GetSystemTimeAsFileTime()"; + info->monotonic = 0; + ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, + &isTimeAdjustmentDisabled); + if (!ok) { + PyErr_SetFromWindowsErr(0); + return -1; + } + info->resolution = timeIncrement * 1e-7; + info->adjustable = 1; + } + +#else /* MS_WINDOWS */ + int err; +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; +#else + struct timeval tv; +#endif + + assert(info == NULL || raise); + +#ifdef HAVE_CLOCK_GETTIME + err = clock_gettime(CLOCK_REALTIME, &ts); + if (err) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + if (_PyTime_FromTimespec(tp, &ts, raise) < 0) + return -1; + + if (info) { + struct timespec res; + info->implementation = "clock_gettime(CLOCK_REALTIME)"; + info->monotonic = 0; + info->adjustable = 1; + if (clock_getres(CLOCK_REALTIME, &res) == 0) + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + else + info->resolution = 1e-9; + } +#else /* HAVE_CLOCK_GETTIME */ + + /* test gettimeofday() */ +#ifdef GETTIMEOFDAY_NO_TZ + err = gettimeofday(&tv); +#else + err = gettimeofday(&tv, (struct timezone *)NULL); +#endif + if (err) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + if (_PyTime_FromTimeval(tp, &tv, raise) < 0) + return -1; + + if (info) { + info->implementation = "gettimeofday()"; + info->resolution = 1e-6; + info->monotonic = 0; + info->adjustable = 1; + } +#endif /* !HAVE_CLOCK_GETTIME */ +#endif /* !MS_WINDOWS */ + return 0; +} + +_PyTime_t +_PyTime_GetSystemClock(void) +{ + _PyTime_t t; + if (pygettimeofday(&t, NULL, 0) < 0) { + /* should not happen, _PyTime_Init() checked the clock at startup */ + assert(0); + + /* use a fixed value instead of a random value from the stack */ + t = 0; + } + return t; +} + +int +_PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) +{ + return pygettimeofday(t, info, 1); +} + + +static int +pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) +{ +#if defined(MS_WINDOWS) + ULONGLONG result; + + assert(info == NULL || raise); + + result = GetTickCount64(); + + *tp = result * MS_TO_NS; + if (*tp / MS_TO_NS != result) { + if (raise) { + _PyTime_overflow(); + return -1; + } + /* Hello, time traveler! */ + assert(0); + } + + if (info) { + DWORD timeAdjustment, timeIncrement; + BOOL isTimeAdjustmentDisabled, ok; + info->implementation = "GetTickCount64()"; + info->monotonic = 1; + ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, + &isTimeAdjustmentDisabled); + if (!ok) { + PyErr_SetFromWindowsErr(0); + return -1; + } + info->resolution = timeIncrement * 1e-7; + info->adjustable = 0; + } + +#elif defined(__APPLE__) + static mach_timebase_info_data_t timebase; + uint64_t time; + + if (timebase.denom == 0) { + /* According to the Technical Q&A QA1398, mach_timebase_info() cannot + fail: https://developer.apple.com/library/mac/#qa/qa1398/ */ + (void)mach_timebase_info(&timebase); + } + + time = mach_absolute_time(); + + /* apply timebase factor */ + time *= timebase.numer; + time /= timebase.denom; + + *tp = time; + + if (info) { + info->implementation = "mach_absolute_time()"; + info->resolution = (double)timebase.numer / timebase.denom * 1e-9; + info->monotonic = 1; + info->adjustable = 0; + } + +#else + struct timespec ts; +#ifdef CLOCK_HIGHRES + const clockid_t clk_id = CLOCK_HIGHRES; + const char *implementation = "clock_gettime(CLOCK_HIGHRES)"; +#else + const clockid_t clk_id = (clockid_t)4; + const char *implementation = "clock_gettime((clockid_t)4)"; +#endif + + assert(info == NULL || raise); + + if (clock_gettime(clk_id, &ts) != 0) { + if (raise) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + return -1; + } + + if (info) { + struct timespec res; + info->monotonic = 1; + info->implementation = implementation; + info->adjustable = 0; + if (clock_getres(clk_id, &res) != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + } + if (_PyTime_FromTimespec(tp, &ts, raise) < 0) + return -1; +#endif + return 0; +} + +_PyTime_t +_PyTime_GetMonotonicClock(void) +{ + _PyTime_t t; + if (pymonotonic(&t, NULL, 0) < 0) { + /* should not happen, _PyTime_Init() checked that monotonic clock at + startup */ + assert(0); + + /* use a fixed value instead of a random value from the stack */ + t = 0; + } + return t; +} + +int +_PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +{ + return pymonotonic(tp, info, 1); +} + +int +_PyTime_Init(void) +{ + _PyTime_t t; + + /* ensure that the system clock works */ + if (_PyTime_GetSystemClockWithInfo(&t, NULL) < 0) + return -1; + + /* ensure that the operating system provides a monotonic clock */ + if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) < 0) + return -1; + + /* check that _PyTime_FromSeconds() cannot overflow */ + assert(INT_MAX <= _PyTime_MAX / SEC_TO_NS); + assert(INT_MIN >= _PyTime_MIN / SEC_TO_NS); + return 0; +} From c5da11c42dfe78efdfea423d51a3b58472d23c57 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 00:10:20 +0000 Subject: [PATCH 14/72] Use pygettimeofday to replace pymonotonic implementation --- python_config/pytime.c | 108 +---------------------------------------- 1 file changed, 1 insertion(+), 107 deletions(-) diff --git a/python_config/pytime.c b/python_config/pytime.c index 37fa5cd..7e46e46 100644 --- a/python_config/pytime.c +++ b/python_config/pytime.c @@ -561,113 +561,7 @@ _PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) static int pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) { -#if defined(MS_WINDOWS) - ULONGLONG result; - - assert(info == NULL || raise); - - result = GetTickCount64(); - - *tp = result * MS_TO_NS; - if (*tp / MS_TO_NS != result) { - if (raise) { - _PyTime_overflow(); - return -1; - } - /* Hello, time traveler! */ - assert(0); - } - - if (info) { - DWORD timeAdjustment, timeIncrement; - BOOL isTimeAdjustmentDisabled, ok; - info->implementation = "GetTickCount64()"; - info->monotonic = 1; - ok = GetSystemTimeAdjustment(&timeAdjustment, &timeIncrement, - &isTimeAdjustmentDisabled); - if (!ok) { - PyErr_SetFromWindowsErr(0); - return -1; - } - info->resolution = timeIncrement * 1e-7; - info->adjustable = 0; - } - -#elif defined(__APPLE__) - static mach_timebase_info_data_t timebase; - uint64_t time; - - if (timebase.denom == 0) { - /* According to the Technical Q&A QA1398, mach_timebase_info() cannot - fail: https://developer.apple.com/library/mac/#qa/qa1398/ */ - (void)mach_timebase_info(&timebase); - } - - time = mach_absolute_time(); - - /* apply timebase factor */ - time *= timebase.numer; - time /= timebase.denom; - - *tp = time; - - if (info) { - info->implementation = "mach_absolute_time()"; - info->resolution = (double)timebase.numer / timebase.denom * 1e-9; - info->monotonic = 1; - info->adjustable = 0; - } - -#else - struct timespec ts; -#ifdef CLOCK_HIGHRES - const clockid_t clk_id = CLOCK_HIGHRES; - const char *implementation = "clock_gettime(CLOCK_HIGHRES)"; -#else - const clockid_t clk_id = (clockid_t)4; - const char *implementation = "clock_gettime((clockid_t)4)"; -#endif - - assert(info == NULL || raise); - - if (clock_gettime(clk_id, &ts) != 0) { - if (raise) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - return -1; - } - - if (info) { - struct timespec res; - info->monotonic = 1; - info->implementation = implementation; - info->adjustable = 0; - if (clock_getres(clk_id, &res) != 0) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; - } - if (_PyTime_FromTimespec(tp, &ts, raise) < 0) - return -1; -#endif - return 0; -} - -_PyTime_t -_PyTime_GetMonotonicClock(void) -{ - _PyTime_t t; - if (pymonotonic(&t, NULL, 0) < 0) { - /* should not happen, _PyTime_Init() checked that monotonic clock at - startup */ - assert(0); - - /* use a fixed value instead of a random value from the stack */ - t = 0; - } - return t; + return pygettimeofday(tp, info, raise); /* should be good enough? */ } int From cc8ddc8e6af4ac0f1d20b48f742131523f3f9342 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 00:19:36 +0000 Subject: [PATCH 15/72] Re-add accidentally deleted function - oops. --- python_config/pytime.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python_config/pytime.c b/python_config/pytime.c index 7e46e46..c3ffaeb 100644 --- a/python_config/pytime.c +++ b/python_config/pytime.c @@ -564,6 +564,21 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) return pygettimeofday(tp, info, raise); /* should be good enough? */ } +_PyTime_t +_PyTime_GetMonotonicClock(void) +{ + _PyTime_t t; + if (pymonotonic(&t, NULL, 0) < 0) { + /* should not happen, _PyTime_Init() checked that monotonic clock at + startup */ + assert(0); + + /* use a fixed value instead of a random value from the stack */ + t = 0; + } + return t; +} + int _PyTime_GetMonotonicClockWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) { From 8d9c9fad320fc4d3f9399d609d106b66f9fd194c Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 00:30:21 +0000 Subject: [PATCH 16/72] Use patched copy of random.c --- python_build/Makefile | 1 + python_config/random.c | 489 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 490 insertions(+) create mode 100644 python_config/random.c diff --git a/python_build/Makefile b/python_build/Makefile index 652e83a..553234a 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -76,6 +76,7 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/random.c $(PYDIR)/Python/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old diff --git a/python_config/random.c b/python_config/random.c new file mode 100644 index 0000000..d203939 --- /dev/null +++ b/python_config/random.c @@ -0,0 +1,489 @@ +#include "Python.h" +#ifdef MS_WINDOWS +# include +#else +# include +# ifdef HAVE_SYS_STAT_H +# include +# endif +# ifdef HAVE_LINUX_RANDOM_H +# include +# endif +# if defined(HAVE_SYS_RANDOM_H) && (defined(HAVE_GETRANDOM) || defined(HAVE_GETENTROPY)) +# include +# endif +# if !defined(HAVE_GETRANDOM) && defined(HAVE_GETRANDOM_SYSCALL) +# include +# endif +#endif + +#ifdef Py_DEBUG +int _Py_HashSecret_Initialized = 0; +#else +static int _Py_HashSecret_Initialized = 0; +#endif + +#ifdef MS_WINDOWS +static HCRYPTPROV hCryptProv = 0; + +static int +win32_urandom_init(int raise) +{ + /* Acquire context */ + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, + PROV_RSA_FULL, CRYPT_VERIFYCONTEXT)) + goto error; + + return 0; + +error: + if (raise) + PyErr_SetFromWindowsErr(0); + else + Py_FatalError("Failed to initialize Windows random API (CryptoGen)"); + return -1; +} + +/* Fill buffer with size pseudo-random bytes generated by the Windows CryptoGen + API. Return 0 on success, or raise an exception and return -1 on error. */ +static int +win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) +{ + Py_ssize_t chunk; + + if (hCryptProv == 0) + { + if (win32_urandom_init(raise) == -1) + return -1; + } + + while (size > 0) + { + chunk = size > INT_MAX ? INT_MAX : size; + if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer)) + { + /* CryptGenRandom() failed */ + if (raise) + PyErr_SetFromWindowsErr(0); + else + Py_FatalError("Failed to initialized the randomized hash " + "secret using CryptoGen)"); + return -1; + } + buffer += chunk; + size -= chunk; + } + return 0; +} + +/* Issue #25003: Don't use getentropy() on Solaris (available since + * Solaris 11.3), it is blocking whereas os.urandom() should not block. */ +#elif defined(HAVE_GETENTROPY) && !defined(sun) +#define PY_GETENTROPY 1 + +/* Fill buffer with size pseudo-random bytes generated by getentropy(). + Return 0 on success, or raise an exception and return -1 on error. + + If fatal is nonzero, call Py_FatalError() instead of raising an exception + on error. */ +static int +py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal) +{ + while (size > 0) { + Py_ssize_t len = Py_MIN(size, 256); + int res; + + if (!fatal) { + Py_BEGIN_ALLOW_THREADS + res = getentropy(buffer, len); + Py_END_ALLOW_THREADS + + if (res < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + } + else { + res = getentropy(buffer, len); + if (res < 0) + Py_FatalError("getentropy() failed"); + } + + buffer += len; + size -= len; + } + return 0; +} + +#else + +#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) +#define PY_GETRANDOM 1 + +/* Call getrandom() + - Return 1 on success + - Return 0 if getrandom() syscall is not available (failed with ENOSYS or + EPERM) or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom + not initialized yet) and raise=0. + - Raise an exception (if raise is non-zero) and return -1 on error: + getrandom() failed with EINTR and the Python signal handler raised an + exception, or getrandom() failed with a different error. */ +static int +py_getrandom(void *buffer, Py_ssize_t size, int raise) +{ + /* Is getrandom() supported by the running kernel? Set to 0 if getrandom() + failed with ENOSYS or EPERM. Need Linux kernel 3.17 or newer, or Solaris + 11.3 or newer */ + static int getrandom_works = 1; + + /* getrandom() on Linux will block if called before the kernel has + * initialized the urandom entropy pool. This will cause Python + * to hang on startup if called very early in the boot process - + * see https://bugs.python.org/issue26839. To avoid this, use the + * GRND_NONBLOCK flag. */ + const int flags = GRND_NONBLOCK; + long n; + + if (!getrandom_works) { + return 0; + } + + while (0 < size) { +#ifdef sun + /* Issue #26735: On Solaris, getrandom() is limited to returning up + to 1024 bytes */ + n = Py_MIN(size, 1024); +#else + n = Py_MIN(size, LONG_MAX); +#endif + + errno = 0; +#ifdef HAVE_GETRANDOM + if (raise) { + Py_BEGIN_ALLOW_THREADS + n = getrandom(buffer, n, flags); + Py_END_ALLOW_THREADS + } + else { + n = getrandom(buffer, n, flags); + } +#else + /* On Linux, use the syscall() function because the GNU libc doesn't + * expose the Linux getrandom() syscall yet. See: + * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ + if (raise) { + Py_BEGIN_ALLOW_THREADS + n = syscall(SYS_getrandom, buffer, n, flags); + Py_END_ALLOW_THREADS + } + else { + n = syscall(SYS_getrandom, buffer, n, flags); + } +#endif + + if (n < 0) { + /* ENOSYS: getrandom() syscall not supported by the kernel (but + * maybe supported by the host which built Python). EPERM: + * getrandom() syscall blocked by SECCOMP or something else. */ + if (errno == ENOSYS || errno == EPERM) { + getrandom_works = 0; + return 0; + } + if (errno == EAGAIN) { + /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system + urandom is not initialiazed yet. In this case, fall back on + reading from /dev/urandom. + + Note: In this case the data read will not be random so + should not be used for cryptographic purposes. Retaining + the existing semantics for practical purposes. */ + getrandom_works = 0; + return 0; + } + + if (errno == EINTR) { + if (PyErr_CheckSignals()) { + if (!raise) { + Py_FatalError("getrandom() interrupted by a signal"); + } + return -1; + } + + /* retry getrandom() */ + continue; + } + + if (raise) { + PyErr_SetFromErrno(PyExc_OSError); + } + else { + Py_FatalError("getrandom() failed"); + } + return -1; + } + + buffer += n; + size -= n; + } + return 1; +} +#endif + +static struct { + int fd; + dev_t st_dev; + ino_t st_ino; +} urandom_cache = { -1 }; + + +/* Read 'size' random bytes from py_getrandom(). Fall back on reading from + /dev/urandom if getrandom() is not available. + + Call Py_FatalError() on error. */ +static void +dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) +{ + int fd; + Py_ssize_t n; + + assert (0 < size); + +#ifdef PY_GETRANDOM + if (py_getrandom(buffer, size, 0) == 1) { + return; + } + /* getrandom() failed with ENOSYS or EPERM, + fall back on reading /dev/urandom */ +#endif + + fd = _Py_open_noraise("/dev/urandom", O_RDONLY); + if (fd < 0) { + Py_FatalError("Failed to open /dev/urandom"); + } + + while (0 < size) + { + do { + n = read(fd, buffer, (size_t)size); + } while (n < 0 && errno == EINTR); + + if (n <= 0) { + /* read() failed or returned 0 bytes */ + Py_FatalError("Failed to read bytes from /dev/urandom"); + break; + } + buffer += n; + size -= n; + } + close(fd); +} + +/* Read 'size' random bytes from py_getrandom(). Fall back on reading from + /dev/urandom if getrandom() is not available. + + Return 0 on success. Raise an exception and return -1 on error. */ +static int +dev_urandom_python(char *buffer, Py_ssize_t size) +{ + int fd; + Py_ssize_t n; + struct _Py_stat_struct st; +#ifdef PY_GETRANDOM + int res; +#endif + + if (size <= 0) + return 0; + +#ifdef PY_GETRANDOM + res = py_getrandom(buffer, size, 1); + if (res < 0) { + return -1; + } + if (res == 1) { + return 0; + } + /* getrandom() failed with ENOSYS or EPERM, + fall back on reading /dev/urandom */ +#endif + + if (urandom_cache.fd >= 0) { + /* Does the fd point to the same thing as before? (issue #21207) */ + if (_Py_fstat_noraise(urandom_cache.fd, &st) + || st.st_dev != urandom_cache.st_dev + || st.st_ino != urandom_cache.st_ino) { + /* Something changed: forget the cached fd (but don't close it, + since it probably points to something important for some + third-party code). */ + urandom_cache.fd = -1; + } + } + if (urandom_cache.fd >= 0) + fd = urandom_cache.fd; + else { + fd = _Py_open("/dev/urandom", O_RDONLY); + if (fd < 0) { + if (errno == ENOENT || errno == ENXIO || + errno == ENODEV || errno == EACCES) + PyErr_SetString(PyExc_NotImplementedError, + "/dev/urandom (or equivalent) not found"); + /* otherwise, keep the OSError exception raised by _Py_open() */ + return -1; + } + if (urandom_cache.fd >= 0) { + /* urandom_fd was initialized by another thread while we were + not holding the GIL, keep it. */ + close(fd); + fd = urandom_cache.fd; + } + else { + if (_Py_fstat(fd, &st)) { + close(fd); + return -1; + } + else { + urandom_cache.fd = fd; + urandom_cache.st_dev = st.st_dev; + urandom_cache.st_ino = st.st_ino; + } + } + } + + do { + n = _Py_read(fd, buffer, (size_t)size); + if (n == -1) { + return -1; + } + if (n == 0) { + PyErr_Format(PyExc_RuntimeError, + "Failed to read %zi bytes from /dev/urandom", + size); + return -1; + } + + buffer += n; + size -= n; + } while (0 < size); + + return 0; +} + +static void +dev_urandom_close(void) +{ + if (urandom_cache.fd >= 0) { + close(urandom_cache.fd); + urandom_cache.fd = -1; + } +} + +#endif + +/* Fill buffer with pseudo-random bytes generated by a linear congruent + generator (LCG): + + x(n+1) = (x(n) * 214013 + 2531011) % 2^32 + + Use bits 23..16 of x(n) to generate a byte. */ +static void +lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size) +{ + size_t index; + unsigned int x; + + x = x0; + for (index=0; index < size; index++) { + x *= 214013; + x += 2531011; + /* modulo 2 ^ (8 * sizeof(int)) */ + buffer[index] = (x >> 16) & 0xff; + } +} + +/* Fill buffer with size pseudo-random bytes from the operating system random + number generator (RNG). It is suitable for most cryptographic purposes + except long living private keys for asymmetric encryption. + + Return 0 on success, raise an exception and return -1 on error. */ +int +_PyOS_URandom(void *buffer, Py_ssize_t size) +{ + if (size < 0) { + PyErr_Format(PyExc_ValueError, + "negative argument not allowed"); + return -1; + } + if (size == 0) + return 0; + +#ifdef MS_WINDOWS + return win32_urandom((unsigned char *)buffer, size, 1); +#elif defined(PY_GETENTROPY) + return py_getentropy(buffer, size, 0); +#else + return dev_urandom_python((char*)buffer, size); +#endif +} + +void +_PyRandom_Init(void) +{ + char *env; + unsigned char *secret = (unsigned char *)&_Py_HashSecret.uc; + Py_ssize_t secret_size = sizeof(_Py_HashSecret_t); + assert(secret_size == sizeof(_Py_HashSecret.uc)); + + if (_Py_HashSecret_Initialized) + return; + _Py_HashSecret_Initialized = 1; + + /* + Hash randomization is enabled. Generate a per-process secret, + using PYTHONHASHSEED if provided. + */ + + env = Py_GETENV("PYTHONHASHSEED"); + if (env && *env != '\0' && strcmp(env, "random") != 0) { + char *endptr = env; + unsigned long seed; + seed = strtoul(env, &endptr, 10); + if (*endptr != '\0' + || seed > 4294967295UL + || (errno == ERANGE && seed == ULONG_MAX)) + { + Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer " + "in range [0; 4294967295]"); + } + if (seed == 0) { + /* disable the randomized hash */ + memset(secret, 0, secret_size); + } + else { + lcg_urandom(seed, secret, secret_size); + } + } + else { +#ifdef MS_WINDOWS + (void)win32_urandom(secret, secret_size, 0); +#elif defined(PY_GETENTROPY) + (void)py_getentropy(secret, secret_size, 1); +#else + dev_urandom_noraise(secret, secret_size); +#endif + } +} + +void +_PyRandom_Fini(void) +{ +#ifdef MS_WINDOWS + if (hCryptProv) { + CryptReleaseContext(hCryptProv, 0); + hCryptProv = 0; + } +#elif defined(PY_GETENTROPY) + /* nothing to clean */ +#else + dev_urandom_close(); +#endif +} From 355ac161c0700df4db78bf36fc139fb6fb54c118 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 02:54:45 +0000 Subject: [PATCH 17/72] UNTESTED: GetRandomBytes implementation --- python_build/Makefile | 2 +- python_config/random.c | 81 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 553234a..86eb33a 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -86,7 +86,7 @@ soospatchPY: compilePY cp $(PYDIR)/Python/pytime.c $(PYDIR)/Python/pytime.c_old cat $(PYDIR)/Python/pytime.c_old | sed 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c cp $(PYDIR)/Makefile $(PYDIR)/Makefile_old - cat $(PYDIR)/Makefile_old | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile + cat $(PYDIR)/Makefile_old | sed 's/-Werror=declaration-after-statement//' | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile touch soospatchPY compilePY: extractedPY patchPY diff --git a/python_config/random.c b/python_config/random.c index d203939..bd1107f 100644 --- a/python_config/random.c +++ b/python_config/random.c @@ -17,12 +17,89 @@ # endif #endif +#ifdef SWITCH +#include "switch/types.h" +#include "switch/result.h" +#include "switch/ipc.h" +#include "switch/services/sm.h" +#endif + #ifdef Py_DEBUG int _Py_HashSecret_Initialized = 0; #else static int _Py_HashSecret_Initialized = 0; #endif +#ifdef SWITCH +static Service g_csrngSrv; + +static int +switch_urandom_init(int raise) +{ + Result rc; + + rc = smGetService(&g_csrngSrv, "csrng"); + if (R_SUCCEEDED(rc)) + return 0; + + if (raise) + PyErr_SetString(PyExc_RuntimeError, + "SWITCH: Failed to get csrng service handle."); + else + Py_FatalError("SWITCH: Failed to get csrng service handle."); + return -1; +} + +static int +switch_urandom(unsigned char *buffer, Py_ssize_t size, int raise) +{ + IpcCommand c; + IpcParsedCommand r; + Result rc; + + struct { + u64 magic; + u64 cmd_id; + } *raw; + + struct { + u64 magic; + u64 result; + } *resp; + + if (g_csrngSrv.handle == 0) + switch_urandom_init(raise); + + ipcInitialize(&c); + ipcAddRecvBuffer(&c, buffer, size, 0); + + raw = ipcPrepareHeader(&c, sizeof(*raw)); + raw->magic = SFCI_MAGIC; + raw->cmd_id = 0; + + rc = serviceIpcDispatch(&g_csrngSrv); + + if (!R_SUCCEEDED(rc)) + goto error; + + ipcParse(&r); + + resp = r.Raw; + rc = resp->result; + + if (R_SUCCEEDED(rc)) + return 0; + +error: + if (raise) + PyErr_SetString(PyExc_RuntimeError, + "SWITCH: GetRandomBytes failed"); + else + Py_FatalError("SWITCH: GetRandomBytes failed"); + return -1; +} +#endif + #ifdef MS_WINDOWS static HCRYPTPROV hCryptProv = 0; @@ -418,6 +495,8 @@ _PyOS_URandom(void *buffer, Py_ssize_t size) #ifdef MS_WINDOWS return win32_urandom((unsigned char *)buffer, size, 1); +#elif defined(SWITCH) + return switch_urandom(buffer, size, 0); #elif defined(PY_GETENTROPY) return py_getentropy(buffer, size, 0); #else @@ -465,6 +544,8 @@ _PyRandom_Init(void) else { #ifdef MS_WINDOWS (void)win32_urandom(secret, secret_size, 0); +#elif defined(SWITCH) + (void)switch_urandom(secret, secret_size, 0); #elif defined(PY_GETENTROPY) (void)py_getentropy(secret, secret_size, 1); #else From 141ca1fbb79cf96a86d3368515e12c43aabf3691 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 19:29:11 +0000 Subject: [PATCH 18/72] Use patched copy of fileutils.c --- python_build/Makefile | 1 + python_config/fileutils.c | 1683 +++++++++++++++++++++++++++++++++++++ 2 files changed, 1684 insertions(+) create mode 100644 python_config/fileutils.c diff --git a/python_build/Makefile b/python_build/Makefile index 86eb33a..a4cc674 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -77,6 +77,7 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/random.c $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Python/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old diff --git a/python_config/fileutils.c b/python_config/fileutils.c new file mode 100644 index 0000000..23eed71 --- /dev/null +++ b/python_config/fileutils.c @@ -0,0 +1,1683 @@ +#include "Python.h" +#include "osdefs.h" +#include + +#ifdef MS_WINDOWS +# include +# include +extern int winerror_to_errno(int); +#endif + +#ifdef HAVE_LANGINFO_H +#include +#endif + +#ifdef HAVE_SYS_IOCTL_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif /* HAVE_FCNTL_H */ + +#ifdef __APPLE__ +extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); +#endif + +#ifdef O_CLOEXEC +/* Does open() support the O_CLOEXEC flag? Possible values: + + -1: unknown + 0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23 + 1: open() supports O_CLOEXEC flag, close-on-exec is set + + The flag is used by _Py_open(), _Py_open_noraise(), io.FileIO + and os.open(). */ +int _Py_open_cloexec_works = -1; +#endif + +PyObject * +_Py_device_encoding(int fd) +{ +#if defined(MS_WINDOWS) + UINT cp; +#endif + int valid; + _Py_BEGIN_SUPPRESS_IPH + valid = _PyVerify_fd(fd) && isatty(fd); + _Py_END_SUPPRESS_IPH + if (!valid) + Py_RETURN_NONE; + +#if defined(MS_WINDOWS) + if (fd == 0) + cp = GetConsoleCP(); + else if (fd == 1 || fd == 2) + cp = GetConsoleOutputCP(); + else + cp = 0; + /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application + has no console */ + if (cp != 0) + return PyUnicode_FromFormat("cp%u", (unsigned int)cp); +#elif defined(CODESET) + { + char *codeset = nl_langinfo(CODESET); + if (codeset != NULL && codeset[0] != 0) + return PyUnicode_FromString(codeset); + } +#endif + Py_RETURN_NONE; +} + +#if !defined(__APPLE__) && !defined(MS_WINDOWS) +extern int _Py_normalize_encoding(const char *, char *, size_t); + +/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale. + On these operating systems, nl_langinfo(CODESET) announces an alias of the + ASCII encoding, whereas mbstowcs() and wcstombs() functions use the + ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use + locale.getpreferredencoding() codec. For example, if command line arguments + are decoded by mbstowcs() and encoded back by os.fsencode(), we get a + UnicodeEncodeError instead of retrieving the original byte string. + + The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C", + nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least + one byte in range 0x80-0xff can be decoded from the locale encoding. The + workaround is also enabled on error, for example if getting the locale + failed. + + Values of force_ascii: + + 1: the workaround is used: Py_EncodeLocale() uses + encode_ascii_surrogateescape() and Py_DecodeLocale() uses + decode_ascii_surrogateescape() + 0: the workaround is not used: Py_EncodeLocale() uses wcstombs() and + Py_DecodeLocale() uses mbstowcs() + -1: unknown, need to call check_force_ascii() to get the value +*/ +static int force_ascii = -1; + +static int +check_force_ascii(void) +{ + char *loc; +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + char *codeset, **alias; + char encoding[100]; + int is_ascii; + unsigned int i; + char* ascii_aliases[] = { + "ascii", + "646", + "ansi-x3.4-1968", + "ansi-x3-4-1968", + "ansi-x3.4-1986", + "cp367", + "csascii", + "ibm367", + "iso646-us", + "iso-646.irv-1991", + "iso-ir-6", + "us", + "us-ascii", + NULL + }; +#endif + + loc = setlocale(LC_CTYPE, NULL); + if (loc == NULL) + goto error; + if (strcmp(loc, "C") != 0) { + /* the LC_CTYPE locale is different than C */ + return 0; + } + +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + /* CODESET is not set or empty */ + goto error; + } + if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) + goto error; + + is_ascii = 0; + for (alias=ascii_aliases; *alias != NULL; alias++) { + if (strcmp(encoding, *alias) == 0) { + is_ascii = 1; + break; + } + } + if (!is_ascii) { + /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */ + return 0; + } + + for (i=0x80; i<0xff; i++) { + unsigned char ch; + wchar_t wch; + size_t res; + + ch = (unsigned char)i; + res = mbstowcs(&wch, (char*)&ch, 1); + if (res != (size_t)-1) { + /* decoding a non-ASCII character from the locale encoding succeed: + the locale encoding is not ASCII, force ASCII */ + return 1; + } + } + /* None of the bytes in the range 0x80-0xff can be decoded from the locale + encoding: the locale encoding is really ASCII */ + return 0; +#else + /* nl_langinfo(CODESET) is not available: always force ASCII */ + return 1; +#endif + +error: + /* if an error occurred, force the ASCII encoding */ + return 1; +} + +static char* +encode_ascii_surrogateescape(const wchar_t *text, size_t *error_pos) +{ + char *result = NULL, *out; + size_t len, i; + wchar_t ch; + + if (error_pos != NULL) + *error_pos = (size_t)-1; + + len = wcslen(text); + + result = PyMem_Malloc(len + 1); /* +1 for NUL byte */ + if (result == NULL) + return NULL; + + out = result; + for (i=0; i PY_SSIZE_T_MAX/sizeof(wchar_t)) + return NULL; + res = PyMem_RawMalloc(argsize*sizeof(wchar_t)); + if (!res) + return NULL; + + in = (unsigned char*)arg; + out = res; + while(*in) + if(*in < 128) + *out++ = *in++; + else + *out++ = 0xdc00 + *in++; + *out = 0; + if (size != NULL) + *size = out - res; + return res; +} +#endif + + +/* Decode a byte string from the locale encoding with the + surrogateescape error handler: undecodable bytes are decoded as characters + in range U+DC80..U+DCFF. If a byte sequence can be decoded as a surrogate + character, escape the bytes using the surrogateescape error handler instead + of decoding them. + + Return a pointer to a newly allocated wide character string, use + PyMem_RawFree() to free the memory. If size is not NULL, write the number of + wide characters excluding the null character into *size + + Return NULL on decoding error or memory allocation error. If *size* is not + NULL, *size is set to (size_t)-1 on memory error or set to (size_t)-2 on + decoding error. + + Decoding errors should never happen, unless there is a bug in the C + library. + + Use the Py_EncodeLocale() function to encode the character string back to a + byte string. */ +wchar_t* +Py_DecodeLocale(const char* arg, size_t *size) +{ +#ifdef __APPLE__ + wchar_t *wstr; + wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); + if (size != NULL) { + if (wstr != NULL) + *size = wcslen(wstr); + else + *size = (size_t)-1; + } + return wstr; +#else + wchar_t *res; + size_t argsize; + size_t count; +#ifdef HAVE_MBRTOWC + unsigned char *in; + wchar_t *out; + mbstate_t mbs; +#endif + +#ifndef MS_WINDOWS + if (force_ascii == -1) + force_ascii = check_force_ascii(); + + if (force_ascii) { + /* force ASCII encoding to workaround mbstowcs() issue */ + res = decode_ascii_surrogateescape(arg, size); + if (res == NULL) + goto oom; + return res; + } +#endif + +#ifdef HAVE_BROKEN_MBSTOWCS + /* Some platforms have a broken implementation of + * mbstowcs which does not count the characters that + * would result from conversion. Use an upper bound. + */ + argsize = strlen(arg); +#else + argsize = mbstowcs(NULL, arg, 0); +#endif + if (argsize != (size_t)-1) { + if (argsize == PY_SSIZE_T_MAX) + goto oom; + argsize += 1; + if (argsize > PY_SSIZE_T_MAX/sizeof(wchar_t)) + goto oom; + res = (wchar_t *)PyMem_RawMalloc(argsize*sizeof(wchar_t)); + if (!res) + goto oom; + count = mbstowcs(res, arg, argsize); + if (count != (size_t)-1) { + wchar_t *tmp; + /* Only use the result if it contains no + surrogate characters. */ + for (tmp = res; *tmp != 0 && + !Py_UNICODE_IS_SURROGATE(*tmp); tmp++) + ; + if (*tmp == 0) { + if (size != NULL) + *size = count; + return res; + } + } + PyMem_RawFree(res); + } + /* Conversion failed. Fall back to escaping with surrogateescape. */ +#ifdef HAVE_MBRTOWC + /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */ + + /* Overallocate; as multi-byte characters are in the argument, the + actual output could use less memory. */ + argsize = strlen(arg) + 1; + if (argsize > PY_SSIZE_T_MAX/sizeof(wchar_t)) + goto oom; + res = (wchar_t*)PyMem_RawMalloc(argsize*sizeof(wchar_t)); + if (!res) + goto oom; + in = (unsigned char*)arg; + out = res; + memset(&mbs, 0, sizeof mbs); + while (argsize) { + size_t converted = mbrtowc(out, (char*)in, argsize, &mbs); + if (converted == 0) + /* Reached end of string; null char stored. */ + break; + if (converted == (size_t)-2) { + /* Incomplete character. This should never happen, + since we provide everything that we have - + unless there is a bug in the C library, or I + misunderstood how mbrtowc works. */ + PyMem_RawFree(res); + if (size != NULL) + *size = (size_t)-2; + return NULL; + } + if (converted == (size_t)-1) { + /* Conversion error. Escape as UTF-8b, and start over + in the initial shift state. */ + *out++ = 0xdc00 + *in++; + argsize--; + memset(&mbs, 0, sizeof mbs); + continue; + } + if (Py_UNICODE_IS_SURROGATE(*out)) { + /* Surrogate character. Escape the original + byte sequence with surrogateescape. */ + argsize -= converted; + while (converted--) + *out++ = 0xdc00 + *in++; + continue; + } + /* successfully converted some bytes */ + in += converted; + argsize -= converted; + out++; + } + if (size != NULL) + *size = out - res; +#else /* HAVE_MBRTOWC */ + /* Cannot use C locale for escaping; manually escape as if charset + is ASCII (i.e. escape all bytes > 128. This will still roundtrip + correctly in the locale's charset, which must be an ASCII superset. */ + res = decode_ascii_surrogateescape(arg, size); + if (res == NULL) + goto oom; +#endif /* HAVE_MBRTOWC */ + return res; +oom: + if (size != NULL) + *size = (size_t)-1; + return NULL; +#endif /* __APPLE__ */ +} + +/* Encode a wide character string to the locale encoding with the + surrogateescape error handler: surrogate characters in the range + U+DC80..U+DCFF are converted to bytes 0x80..0xFF. + + Return a pointer to a newly allocated byte string, use PyMem_Free() to free + the memory. Return NULL on encoding or memory allocation error. + + If error_pos is not NULL, *error_pos is set to the index of the invalid + character on encoding error, or set to (size_t)-1 otherwise. + + Use the Py_DecodeLocale() function to decode the bytes string back to a wide + character string. */ +char* +Py_EncodeLocale(const wchar_t *text, size_t *error_pos) +{ +#ifdef __APPLE__ + Py_ssize_t len; + PyObject *unicode, *bytes = NULL; + char *cpath; + + unicode = PyUnicode_FromWideChar(text, wcslen(text)); + if (unicode == NULL) + return NULL; + + bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); + Py_DECREF(unicode); + if (bytes == NULL) { + PyErr_Clear(); + if (error_pos != NULL) + *error_pos = (size_t)-1; + return NULL; + } + + len = PyBytes_GET_SIZE(bytes); + cpath = PyMem_Malloc(len+1); + if (cpath == NULL) { + PyErr_Clear(); + Py_DECREF(bytes); + if (error_pos != NULL) + *error_pos = (size_t)-1; + return NULL; + } + memcpy(cpath, PyBytes_AsString(bytes), len + 1); + Py_DECREF(bytes); + return cpath; +#else /* __APPLE__ */ + const size_t len = wcslen(text); + char *result = NULL, *bytes = NULL; + size_t i, size, converted; + wchar_t c, buf[2]; + +#ifndef MS_WINDOWS + if (force_ascii == -1) + force_ascii = check_force_ascii(); + + if (force_ascii) + return encode_ascii_surrogateescape(text, error_pos); +#endif + + /* The function works in two steps: + 1. compute the length of the output buffer in bytes (size) + 2. outputs the bytes */ + size = 0; + buf[1] = 0; + while (1) { + for (i=0; i < len; i++) { + c = text[i]; + if (c >= 0xdc80 && c <= 0xdcff) { + /* UTF-8b surrogate */ + if (bytes != NULL) { + *bytes++ = c - 0xdc00; + size--; + } + else + size++; + continue; + } + else { + buf[0] = c; + if (bytes != NULL) + converted = wcstombs(bytes, buf, size); + else + converted = wcstombs(NULL, buf, 0); + if (converted == (size_t)-1) { + if (result != NULL) + PyMem_Free(result); + if (error_pos != NULL) + *error_pos = i; + return NULL; + } + if (bytes != NULL) { + bytes += converted; + size -= converted; + } + else + size += converted; + } + } + if (result != NULL) { + *bytes = '\0'; + break; + } + + size += 1; /* nul byte at the end */ + result = PyMem_Malloc(size); + if (result == NULL) { + if (error_pos != NULL) + *error_pos = (size_t)-1; + return NULL; + } + bytes = result; + } + return result; +#endif /* __APPLE__ */ +} + + +#ifdef MS_WINDOWS +static __int64 secs_between_epochs = 11644473600; /* Seconds between 1.1.1601 and 1.1.1970 */ + +static void +FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out) +{ + /* XXX endianness. Shouldn't matter, as all Windows implementations are little-endian */ + /* Cannot simply cast and dereference in_ptr, + since it might not be aligned properly */ + __int64 in; + memcpy(&in, in_ptr, sizeof(in)); + *nsec_out = (int)(in % 10000000) * 100; /* FILETIME is in units of 100 nsec. */ + *time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t); +} + +void +_Py_time_t_to_FILE_TIME(time_t time_in, int nsec_in, FILETIME *out_ptr) +{ + /* XXX endianness */ + __int64 out; + out = time_in + secs_between_epochs; + out = out * 10000000 + nsec_in / 100; + memcpy(out_ptr, &out, sizeof(out)); +} + +/* Below, we *know* that ugo+r is 0444 */ +#if _S_IREAD != 0400 +#error Unsupported C library +#endif +static int +attributes_to_mode(DWORD attr) +{ + int m = 0; + if (attr & FILE_ATTRIBUTE_DIRECTORY) + m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */ + else + m |= _S_IFREG; + if (attr & FILE_ATTRIBUTE_READONLY) + m |= 0444; + else + m |= 0666; + return m; +} + +void +_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, + struct _Py_stat_struct *result) +{ + memset(result, 0, sizeof(*result)); + result->st_mode = attributes_to_mode(info->dwFileAttributes); + result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow; + result->st_dev = info->dwVolumeSerialNumber; + result->st_rdev = result->st_dev; + FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); + result->st_nlink = info->nNumberOfLinks; + result->st_ino = (((__int64)info->nFileIndexHigh)<<32) + info->nFileIndexLow; + if (reparse_tag == IO_REPARSE_TAG_SYMLINK) { + /* first clear the S_IFMT bits */ + result->st_mode ^= (result->st_mode & S_IFMT); + /* now set the bits that make this a symlink */ + result->st_mode |= S_IFLNK; + } + result->st_file_attributes = info->dwFileAttributes; +} +#endif + +/* Return information about a file. + + On POSIX, use fstat(). + + On Windows, use GetFileType() and GetFileInformationByHandle() which support + files larger than 2 GB. fstat() may fail with EOVERFLOW on files larger + than 2 GB because the file size type is a signed 32-bit integer: see issue + #23152. + + On Windows, set the last Windows error and return nonzero on error. On + POSIX, set errno and return nonzero on error. Fill status and return 0 on + success. */ +int +_Py_fstat_noraise(int fd, struct _Py_stat_struct *status) +{ +#ifdef MS_WINDOWS + BY_HANDLE_FILE_INFORMATION info; + HANDLE h; + int type; + + if (!_PyVerify_fd(fd)) + h = INVALID_HANDLE_VALUE; + else { + _Py_BEGIN_SUPPRESS_IPH + h = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + } + + if (h == INVALID_HANDLE_VALUE) { + /* errno is already set by _get_osfhandle, but we also set + the Win32 error for callers who expect that */ + SetLastError(ERROR_INVALID_HANDLE); + return -1; + } + memset(status, 0, sizeof(*status)); + + type = GetFileType(h); + if (type == FILE_TYPE_UNKNOWN) { + DWORD error = GetLastError(); + if (error != 0) { + errno = winerror_to_errno(error); + return -1; + } + /* else: valid but unknown file */ + } + + if (type != FILE_TYPE_DISK) { + if (type == FILE_TYPE_CHAR) + status->st_mode = _S_IFCHR; + else if (type == FILE_TYPE_PIPE) + status->st_mode = _S_IFIFO; + return 0; + } + + if (!GetFileInformationByHandle(h, &info)) { + /* The Win32 error is already set, but we also set errno for + callers who expect it */ + errno = winerror_to_errno(GetLastError()); + return -1; + } + + _Py_attribute_data_to_stat(&info, 0, status); + /* specific to fstat() */ + status->st_ino = (((__int64)info.nFileIndexHigh)<<32) + info.nFileIndexLow; + return 0; +#else + return fstat(fd, status); +#endif +} + +/* Return information about a file. + + On POSIX, use fstat(). + + On Windows, use GetFileType() and GetFileInformationByHandle() which support + files larger than 2 GB. fstat() may fail with EOVERFLOW on files larger + than 2 GB because the file size type is a signed 32-bit integer: see issue + #23152. + + Raise an exception and return -1 on error. On Windows, set the last Windows + error on error. On POSIX, set errno on error. Fill status and return 0 on + success. + + Release the GIL to call GetFileType() and GetFileInformationByHandle(), or + to call fstat(). The caller must hold the GIL. */ +int +_Py_fstat(int fd, struct _Py_stat_struct *status) +{ + int res; + + Py_BEGIN_ALLOW_THREADS + res = _Py_fstat_noraise(fd, status); + Py_END_ALLOW_THREADS + + if (res != 0) { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErr(0); +#else + PyErr_SetFromErrno(PyExc_OSError); +#endif + return -1; + } + return 0; +} + +/* Call _wstat() on Windows, or encode the path to the filesystem encoding and + call stat() otherwise. Only fill st_mode attribute on Windows. + + Return 0 on success, -1 on _wstat() / stat() error, -2 if an exception was + raised. */ + +int +_Py_stat(PyObject *path, struct stat *statbuf) +{ +#ifdef MS_WINDOWS + int err; + struct _stat wstatbuf; + wchar_t *wpath; + + wpath = PyUnicode_AsUnicode(path); + if (wpath == NULL) + return -2; + err = _wstat(wpath, &wstatbuf); + if (!err) + statbuf->st_mode = wstatbuf.st_mode; + return err; +#else + int ret; + PyObject *bytes = PyUnicode_EncodeFSDefault(path); + if (bytes == NULL) + return -2; + ret = stat(PyBytes_AS_STRING(bytes), statbuf); + Py_DECREF(bytes); + return ret; +#endif +} + + +static int +get_inheritable(int fd, int raise) +{ +#ifdef MS_WINDOWS + HANDLE handle; + DWORD flags; + + if (!_PyVerify_fd(fd)) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + _Py_BEGIN_SUPPRESS_IPH + handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + if (handle == INVALID_HANDLE_VALUE) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (!GetHandleInformation(handle, &flags)) { + if (raise) + PyErr_SetFromWindowsErr(0); + return -1; + } + + return (flags & HANDLE_FLAG_INHERIT); +#else + int flags; + + flags = fcntl(fd, F_GETFD, 0); + if (flags == -1) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + return !(flags & FD_CLOEXEC); +#endif +} + +/* Get the inheritable flag of the specified file descriptor. + Return 1 if the file descriptor can be inherited, 0 if it cannot, + raise an exception and return -1 on error. */ +int +_Py_get_inheritable(int fd) +{ + return get_inheritable(fd, 1); +} + +static int +set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) +{ +#ifdef MS_WINDOWS + HANDLE handle; + DWORD flags; +#else +#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + static int ioctl_works = -1; + int request; + int err; +#endif + int flags; + int res; +#endif + + /* atomic_flag_works can only be used to make the file descriptor + non-inheritable */ + assert(!(atomic_flag_works != NULL && inheritable)); + + if (atomic_flag_works != NULL && !inheritable) { + if (*atomic_flag_works == -1) { + int isInheritable = get_inheritable(fd, raise); + if (isInheritable == -1) + return -1; + *atomic_flag_works = !isInheritable; + } + + if (*atomic_flag_works) + return 0; + } + +#ifdef MS_WINDOWS + if (!_PyVerify_fd(fd)) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + _Py_BEGIN_SUPPRESS_IPH + handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + if (handle == INVALID_HANDLE_VALUE) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (inheritable) + flags = HANDLE_FLAG_INHERIT; + else + flags = 0; + if (!SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags)) { + if (raise) + PyErr_SetFromWindowsErr(0); + return -1; + } + return 0; + +#else + +#if defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + if (ioctl_works != 0) { + /* fast-path: ioctl() only requires one syscall */ + if (inheritable) + request = FIONCLEX; + else + request = FIOCLEX; + err = ioctl(fd, request, NULL); + if (!err) { + ioctl_works = 1; + return 0; + } + + if (errno != ENOTTY && errno != EACCES) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + else { + /* Issue #22258: Here, ENOTTY means "Inappropriate ioctl for + device". The ioctl is declared but not supported by the kernel. + Remember that ioctl() doesn't work. It is the case on + Illumos-based OS for example. + + Issue #27057: When SELinux policy disallows ioctl it will fail + with EACCES. While FIOCLEX is safe operation it may be + unavailable because ioctl was denied altogether. + This can be the case on Android. */ + ioctl_works = 0; + } + /* fallback to fcntl() if ioctl() does not work */ + } +#endif + + /* slow-path: fcntl() requires two syscalls */ + flags = fcntl(fd, F_GETFD); + if (flags < 0) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (inheritable) + flags &= ~FD_CLOEXEC; + else + flags |= FD_CLOEXEC; + res = fcntl(fd, F_SETFD, flags); + if (res < 0) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + return 0; +#endif +} + +/* Make the file descriptor non-inheritable. + Return 0 on success, set errno and return -1 on error. */ +static int +make_non_inheritable(int fd) +{ + return set_inheritable(fd, 0, 0, NULL); +} + +/* Set the inheritable flag of the specified file descriptor. + On success: return 0, on error: raise an exception if raise is nonzero + and return -1. + + If atomic_flag_works is not NULL: + + * if *atomic_flag_works==-1, check if the inheritable is set on the file + descriptor: if yes, set *atomic_flag_works to 1, otherwise set to 0 and + set the inheritable flag + * if *atomic_flag_works==1: do nothing + * if *atomic_flag_works==0: set inheritable flag to False + + Set atomic_flag_works to NULL if no atomic flag was used to create the + file descriptor. + + atomic_flag_works can only be used to make a file descriptor + non-inheritable: atomic_flag_works must be NULL if inheritable=1. */ +int +_Py_set_inheritable(int fd, int inheritable, int *atomic_flag_works) +{ + return set_inheritable(fd, inheritable, 1, atomic_flag_works); +} + +static int +_Py_open_impl(const char *pathname, int flags, int gil_held) +{ + int fd; + int async_err = 0; +#ifndef MS_WINDOWS + int *atomic_flag_works; +#endif + +#ifdef MS_WINDOWS + flags |= O_NOINHERIT; +#elif defined(O_CLOEXEC) + atomic_flag_works = &_Py_open_cloexec_works; + flags |= O_CLOEXEC; +#else + atomic_flag_works = NULL; +#endif + + if (gil_held) { + do { + Py_BEGIN_ALLOW_THREADS + fd = open(pathname, flags); + Py_END_ALLOW_THREADS + } while (fd < 0 + && errno == EINTR && !(async_err = PyErr_CheckSignals())); + if (async_err) + return -1; + if (fd < 0) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, pathname); + return -1; + } + } + else { + fd = open(pathname, flags); + if (fd < 0) + return -1; + } + +#ifndef MS_WINDOWS + if (set_inheritable(fd, 0, gil_held, atomic_flag_works) < 0) { + close(fd); + return -1; + } +#endif + + return fd; +} + +/* Open a file with the specified flags (wrapper to open() function). + Return a file descriptor on success. Raise an exception and return -1 on + error. + + The file descriptor is created non-inheritable. + + When interrupted by a signal (open() fails with EINTR), retry the syscall, + except if the Python signal handler raises an exception. + + Release the GIL to call open(). The caller must hold the GIL. */ +int +_Py_open(const char *pathname, int flags) +{ +#ifdef WITH_THREAD + /* _Py_open() must be called with the GIL held. */ + assert(PyGILState_Check()); +#endif + return _Py_open_impl(pathname, flags, 1); +} + +/* Open a file with the specified flags (wrapper to open() function). + Return a file descriptor on success. Set errno and return -1 on error. + + The file descriptor is created non-inheritable. + + If interrupted by a signal, fail with EINTR. */ +int +_Py_open_noraise(const char *pathname, int flags) +{ + return _Py_open_impl(pathname, flags, 0); +} + +/* Open a file. Use _wfopen() on Windows, encode the path to the locale + encoding and use fopen() otherwise. + + The file descriptor is created non-inheritable. + + If interrupted by a signal, fail with EINTR. */ +FILE * +_Py_wfopen(const wchar_t *path, const wchar_t *mode) +{ + FILE *f; +#ifndef MS_WINDOWS + char *cpath; + char cmode[10]; + size_t r; + r = wcstombs(cmode, mode, 10); + if (r == (size_t)-1 || r >= 10) { + errno = EINVAL; + return NULL; + } + cpath = Py_EncodeLocale(path, NULL); + if (cpath == NULL) + return NULL; + f = fopen(cpath, cmode); + PyMem_Free(cpath); +#else + f = _wfopen(path, mode); +#endif + if (f == NULL) + return NULL; + if (make_non_inheritable(fileno(f)) < 0) { + fclose(f); + return NULL; + } + return f; +} + +/* Wrapper to fopen(). + + The file descriptor is created non-inheritable. + + If interrupted by a signal, fail with EINTR. */ +FILE* +_Py_fopen(const char *pathname, const char *mode) +{ + FILE *f = fopen(pathname, mode); + if (f == NULL) + return NULL; + if (make_non_inheritable(fileno(f)) < 0) { + fclose(f); + return NULL; + } + return f; +} + +/* Open a file. Call _wfopen() on Windows, or encode the path to the filesystem + encoding and call fopen() otherwise. + + Return the new file object on success. Raise an exception and return NULL + on error. + + The file descriptor is created non-inheritable. + + When interrupted by a signal (open() fails with EINTR), retry the syscall, + except if the Python signal handler raises an exception. + + Release the GIL to call _wfopen() or fopen(). The caller must hold + the GIL. */ +FILE* +_Py_fopen_obj(PyObject *path, const char *mode) +{ + FILE *f; + int async_err = 0; +#ifdef MS_WINDOWS + wchar_t *wpath; + wchar_t wmode[10]; + int usize; + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, + "str file path expected under Windows, got %R", + Py_TYPE(path)); + return NULL; + } + wpath = PyUnicode_AsUnicode(path); + if (wpath == NULL) + return NULL; + + usize = MultiByteToWideChar(CP_ACP, 0, mode, -1, wmode, sizeof(wmode)); + if (usize == 0) { + PyErr_SetFromWindowsErr(0); + return NULL; + } + + do { + Py_BEGIN_ALLOW_THREADS + f = _wfopen(wpath, wmode); + Py_END_ALLOW_THREADS + } while (f == NULL + && errno == EINTR && !(async_err = PyErr_CheckSignals())); +#else + PyObject *bytes; + char *path_bytes; + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + + if (!PyUnicode_FSConverter(path, &bytes)) + return NULL; + path_bytes = PyBytes_AS_STRING(bytes); + + do { + Py_BEGIN_ALLOW_THREADS + f = fopen(path_bytes, mode); + Py_END_ALLOW_THREADS + } while (f == NULL + && errno == EINTR && !(async_err = PyErr_CheckSignals())); + + Py_DECREF(bytes); +#endif + if (async_err) + return NULL; + + if (f == NULL) { + PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path); + return NULL; + } + + if (set_inheritable(fileno(f), 0, 1, NULL) < 0) { + fclose(f); + return NULL; + } + return f; +} + +/* Read count bytes from fd into buf. + + On success, return the number of read bytes, it can be lower than count. + If the current file offset is at or past the end of file, no bytes are read, + and read() returns zero. + + On error, raise an exception, set errno and return -1. + + When interrupted by a signal (read() fails with EINTR), retry the syscall. + If the Python signal handler raises an exception, the function returns -1 + (the syscall is not retried). + + Release the GIL to call read(). The caller must hold the GIL. */ +Py_ssize_t +_Py_read(int fd, void *buf, size_t count) +{ + Py_ssize_t n; + int err; + int async_err = 0; + + /* _Py_read() must not be called with an exception set, otherwise the + * caller may think that read() was interrupted by a signal and the signal + * handler raised an exception. */ + assert(!PyErr_Occurred()); + + if (!_PyVerify_fd(fd)) { + /* save/restore errno because PyErr_SetFromErrno() can modify it */ + err = errno; + PyErr_SetFromErrno(PyExc_OSError); + errno = err; + return -1; + } + +#ifdef MS_WINDOWS + if (count > INT_MAX) { + /* On Windows, the count parameter of read() is an int */ + count = INT_MAX; + } +#else + if (count > PY_SSIZE_T_MAX) { + /* if count is greater than PY_SSIZE_T_MAX, + * read() result is undefined */ + count = PY_SSIZE_T_MAX; + } +#endif + + _Py_BEGIN_SUPPRESS_IPH + do { + Py_BEGIN_ALLOW_THREADS + errno = 0; +#ifdef MS_WINDOWS + n = read(fd, buf, (int)count); +#else + n = read(fd, buf, count); +#endif + /* save/restore errno because PyErr_CheckSignals() + * and PyErr_SetFromErrno() can modify it */ + err = errno; + Py_END_ALLOW_THREADS + } while (n < 0 && err == EINTR && + !(async_err = PyErr_CheckSignals())); + _Py_END_SUPPRESS_IPH + + if (async_err) { + /* read() was interrupted by a signal (failed with EINTR) + * and the Python signal handler raised an exception */ + errno = err; + assert(errno == EINTR && PyErr_Occurred()); + return -1; + } + if (n < 0) { + PyErr_SetFromErrno(PyExc_OSError); + errno = err; + return -1; + } + + return n; +} + +static Py_ssize_t +_Py_write_impl(int fd, const void *buf, size_t count, int gil_held) +{ + Py_ssize_t n; + int err; + int async_err = 0; + + if (!_PyVerify_fd(fd)) { + if (gil_held) { + /* save/restore errno because PyErr_SetFromErrno() can modify it */ + err = errno; + PyErr_SetFromErrno(PyExc_OSError); + errno = err; + } + return -1; + } + + _Py_BEGIN_SUPPRESS_IPH +#ifdef MS_WINDOWS + if (count > 32767 && isatty(fd)) { + /* Issue #11395: the Windows console returns an error (12: not + enough space error) on writing into stdout if stdout mode is + binary and the length is greater than 66,000 bytes (or less, + depending on heap usage). */ + count = 32767; + } + else if (count > INT_MAX) + count = INT_MAX; +#else + if (count > PY_SSIZE_T_MAX) { + /* write() should truncate count to PY_SSIZE_T_MAX, but it's safer + * to do it ourself to have a portable behaviour. */ + count = PY_SSIZE_T_MAX; + } +#endif + + if (gil_held) { + do { + Py_BEGIN_ALLOW_THREADS + errno = 0; +#ifdef MS_WINDOWS + n = write(fd, buf, (int)count); +#else + n = write(fd, buf, count); +#endif + /* save/restore errno because PyErr_CheckSignals() + * and PyErr_SetFromErrno() can modify it */ + err = errno; + Py_END_ALLOW_THREADS + } while (n < 0 && err == EINTR && + !(async_err = PyErr_CheckSignals())); + } + else { + do { + errno = 0; +#ifdef MS_WINDOWS + n = write(fd, buf, (int)count); +#else + n = write(fd, buf, count); +#endif + err = errno; + } while (n < 0 && err == EINTR); + } + _Py_END_SUPPRESS_IPH + + if (async_err) { + /* write() was interrupted by a signal (failed with EINTR) + and the Python signal handler raised an exception (if gil_held is + nonzero). */ + errno = err; + assert(errno == EINTR && (!gil_held || PyErr_Occurred())); + return -1; + } + if (n < 0) { + if (gil_held) + PyErr_SetFromErrno(PyExc_OSError); + errno = err; + return -1; + } + + return n; +} + +/* Write count bytes of buf into fd. + + On success, return the number of written bytes, it can be lower than count + including 0. On error, raise an exception, set errno and return -1. + + When interrupted by a signal (write() fails with EINTR), retry the syscall. + If the Python signal handler raises an exception, the function returns -1 + (the syscall is not retried). + + Release the GIL to call write(). The caller must hold the GIL. */ +Py_ssize_t +_Py_write(int fd, const void *buf, size_t count) +{ + /* _Py_write() must not be called with an exception set, otherwise the + * caller may think that write() was interrupted by a signal and the signal + * handler raised an exception. */ + assert(!PyErr_Occurred()); + + return _Py_write_impl(fd, buf, count, 1); +} + +/* Write count bytes of buf into fd. + * + * On success, return the number of written bytes, it can be lower than count + * including 0. On error, set errno and return -1. + * + * When interrupted by a signal (write() fails with EINTR), retry the syscall + * without calling the Python signal handler. */ +Py_ssize_t +_Py_write_noraise(int fd, const void *buf, size_t count) +{ + return _Py_write_impl(fd, buf, count, 0); +} + +#ifdef HAVE_READLINK + +/* Read value of symbolic link. Encode the path to the locale encoding, decode + the result from the locale encoding. Return -1 on error. */ + +int +_Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz) +{ + char *cpath; + char cbuf[MAXPATHLEN]; + wchar_t *wbuf; + int res; + size_t r1; + + cpath = Py_EncodeLocale(path, NULL); + if (cpath == NULL) { + errno = EINVAL; + return -1; + } + res = (int)readlink(cpath, cbuf, Py_ARRAY_LENGTH(cbuf)); + PyMem_Free(cpath); + if (res == -1) + return -1; + if (res == Py_ARRAY_LENGTH(cbuf)) { + errno = EINVAL; + return -1; + } + cbuf[res] = '\0'; /* buf will be null terminated */ + wbuf = Py_DecodeLocale(cbuf, &r1); + if (wbuf == NULL) { + errno = EINVAL; + return -1; + } + if (bufsiz <= r1) { + PyMem_RawFree(wbuf); + errno = EINVAL; + return -1; + } + wcsncpy(buf, wbuf, bufsiz); + PyMem_RawFree(wbuf); + return (int)r1; +} +#endif + +#ifdef HAVE_REALPATH + +/* Return the canonicalized absolute pathname. Encode path to the locale + encoding, decode the result from the locale encoding. + Return NULL on error. */ + +wchar_t* +_Py_wrealpath(const wchar_t *path, + wchar_t *resolved_path, size_t resolved_path_size) +{ + char *cpath; + char cresolved_path[MAXPATHLEN]; + wchar_t *wresolved_path; + char *res; + size_t r; + cpath = Py_EncodeLocale(path, NULL); + if (cpath == NULL) { + errno = EINVAL; + return NULL; + } + res = realpath(cpath, cresolved_path); + PyMem_Free(cpath); + if (res == NULL) + return NULL; + + wresolved_path = Py_DecodeLocale(cresolved_path, &r); + if (wresolved_path == NULL) { + errno = EINVAL; + return NULL; + } + if (resolved_path_size <= r) { + PyMem_RawFree(wresolved_path); + errno = EINVAL; + return NULL; + } + wcsncpy(resolved_path, wresolved_path, resolved_path_size); + PyMem_RawFree(wresolved_path); + return resolved_path; +} +#endif + +/* Get the current directory. size is the buffer size in wide characters + including the null character. Decode the path from the locale encoding. + Return NULL on error. */ + +wchar_t* +_Py_wgetcwd(wchar_t *buf, size_t size) +{ +#ifdef MS_WINDOWS + int isize = (int)Py_MIN(size, INT_MAX); + return _wgetcwd(buf, isize); +#else + char fname[MAXPATHLEN]; + wchar_t *wname; + size_t len; + + if (getcwd(fname, Py_ARRAY_LENGTH(fname)) == NULL) + return NULL; + wname = Py_DecodeLocale(fname, &len); + if (wname == NULL) + return NULL; + if (size <= len) { + PyMem_RawFree(wname); + return NULL; + } + wcsncpy(buf, wname, size); + PyMem_RawFree(wname); + return buf; +#endif +} + +/* Duplicate a file descriptor. The new file descriptor is created as + non-inheritable. Return a new file descriptor on success, raise an OSError + exception and return -1 on error. + + The GIL is released to call dup(). The caller must hold the GIL. */ +int +_Py_dup(int fd) +{ +#ifdef MS_WINDOWS + HANDLE handle; + DWORD ftype; +#endif + + if (!_PyVerify_fd(fd)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + +#ifdef MS_WINDOWS + _Py_BEGIN_SUPPRESS_IPH + handle = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + /* get the file type, ignore the error if it failed */ + ftype = GetFileType(handle); + + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH + fd = dup(fd); + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + /* Character files like console cannot be make non-inheritable */ + if (ftype != FILE_TYPE_CHAR) { + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + _Py_BEGIN_SUPPRESS_IPH + close(fd); + _Py_END_SUPPRESS_IPH + return -1; + } + } +#elif defined(HAVE_FCNTL_H) && defined(F_DUPFD_CLOEXEC) + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH + fd = fcntl(fd, F_DUPFD_CLOEXEC, 0); + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + +#else + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH + fd = dup(fd); + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + _Py_BEGIN_SUPPRESS_IPH + close(fd); + _Py_END_SUPPRESS_IPH + return -1; + } +#endif + return fd; +} + +#ifndef MS_WINDOWS +/* Get the blocking mode of the file descriptor. + Return 0 if the O_NONBLOCK flag is set, 1 if the flag is cleared, + raise an exception and return -1 on error. */ +int +_Py_get_blocking(int fd) +{ + int flags; + _Py_BEGIN_SUPPRESS_IPH + flags = fcntl(fd, F_GETFL, 0); + _Py_END_SUPPRESS_IPH + if (flags < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + return !(flags & O_NONBLOCK); +} + +/* Set the blocking mode of the specified file descriptor. + + Set the O_NONBLOCK flag if blocking is False, clear the O_NONBLOCK flag + otherwise. + + Return 0 on success, raise an exception and return -1 on error. */ +int +_Py_set_blocking(int fd, int blocking) +{ +#if defined(HAVE_SYS_IOCTL_H) && defined(FIONBIO) + int arg = !blocking; + if (ioctl(fd, FIONBIO, &arg) < 0) + goto error; +#else + int flags, res; + + _Py_BEGIN_SUPPRESS_IPH + flags = fcntl(fd, F_GETFL, 0); + if (flags >= 0) { + if (blocking) + flags = flags & (~O_NONBLOCK); + else + flags = flags | O_NONBLOCK; + + res = fcntl(fd, F_SETFL, flags); + } else { + res = -1; + } + _Py_END_SUPPRESS_IPH + + if (res < 0) + goto error; +#endif + return 0; + +error: + PyErr_SetFromErrno(PyExc_OSError); + return -1; +} +#endif + +#if defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 +/* Legacy implementation of _PyVerify_fd while transitioning to + * MSVC 14.0. This should eventually be removed. (issue23524) + */ + +/* Microsoft CRT in VS2005 and higher will verify that a filehandle is + * valid and raise an assertion if it isn't. + * Normally, an invalid fd is likely to be a C program error and therefore + * an assertion can be useful, but it does contradict the POSIX standard + * which for write(2) states: + * "Otherwise, -1 shall be returned and errno set to indicate the error." + * "[EBADF] The fildes argument is not a valid file descriptor open for + * writing." + * Furthermore, python allows the user to enter any old integer + * as a fd and should merely raise a python exception on error. + * The Microsoft CRT doesn't provide an official way to check for the + * validity of a file descriptor, but we can emulate its internal behaviour + * by using the exported __pinfo data member and knowledge of the + * internal structures involved. + * The structures below must be updated for each version of visual studio + * according to the file internal.h in the CRT source, until MS comes + * up with a less hacky way to do this. + * (all of this is to avoid globally modifying the CRT behaviour using + * _set_invalid_parameter_handler() and _CrtSetReportMode()) + */ +/* The actual size of the structure is determined at runtime. + * Only the first items must be present. + */ +typedef struct { + intptr_t osfhnd; + char osfile; +} my_ioinfo; + +extern __declspec(dllimport) char * __pioinfo[]; +#define IOINFO_L2E 5 +#define IOINFO_ARRAYS 64 +#define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E) +#define _NHANDLE_ (IOINFO_ARRAYS * IOINFO_ARRAY_ELTS) +#define FOPEN 0x01 +#define _NO_CONSOLE_FILENO (intptr_t)-2 + +/* This function emulates what the windows CRT does to validate file handles */ +int +_PyVerify_fd(int fd) +{ + const int i1 = fd >> IOINFO_L2E; + const int i2 = fd & ((1 << IOINFO_L2E) - 1); + + static size_t sizeof_ioinfo = 0; + + /* Determine the actual size of the ioinfo structure, + * as used by the CRT loaded in memory + */ + if (sizeof_ioinfo == 0 && __pioinfo[0] != NULL) { + sizeof_ioinfo = _msize(__pioinfo[0]) / IOINFO_ARRAY_ELTS; + } + if (sizeof_ioinfo == 0) { + /* This should not happen... */ + goto fail; + } + + /* See that it isn't a special CLEAR fileno */ + if (fd != _NO_CONSOLE_FILENO) { + /* Microsoft CRT would check that 0<=fd<_nhandle but we can't do that. Instead + * we check pointer validity and other info + */ + if (0 <= i1 && i1 < IOINFO_ARRAYS && __pioinfo[i1] != NULL) { + /* finally, check that the file is open */ + my_ioinfo* info = (my_ioinfo*)(__pioinfo[i1] + i2 * sizeof_ioinfo); + if (info->osfile & FOPEN) { + return 1; + } + } + } + fail: + errno = EBADF; + return 0; +} + +#endif /* defined _MSC_VER && _MSC_VER >= 1400 && _MSC_VER < 1900 */ From e49bc4786a7b375833c63a48edfbfee470ddf1d0 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 19:38:59 +0000 Subject: [PATCH 19/72] Stub out get_inheritable --- python_config/fileutils.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python_config/fileutils.c b/python_config/fileutils.c index 23eed71..3cc0c07 100644 --- a/python_config/fileutils.c +++ b/python_config/fileutils.c @@ -760,6 +760,8 @@ get_inheritable(int fd, int raise) } return (flags & HANDLE_FLAG_INHERIT); +#elif defined(SWITCH) + return 1; /* Always true */ #else int flags; @@ -798,6 +800,10 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) int res; #endif +#ifdef SWITCH + return 0; /* Always succeed */ +#endif + /* atomic_flag_works can only be used to make the file descriptor non-inheritable */ assert(!(atomic_flag_works != NULL && inheritable)); From 52ca4279d6d2dc19d234d71e8ac0bbfc49477e13 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 19:47:06 +0000 Subject: [PATCH 20/72] Re-enable + to exit option --- source/main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/main.c b/source/main.c index 99f948a..0836cf8 100644 --- a/source/main.c +++ b/source/main.c @@ -55,8 +55,7 @@ int main(int argc, char *argv[]) u32 kDown = hidKeysDown(CONTROLLER_P1_AUTO); - /* Can't crash on exit if there's no exit button :P */ - //if (kDown & KEY_PLUS) break; // break in order to return to hbmenu + if (kDown & KEY_PLUS) break; // break in order to return to hbmenu gfxFlushBuffers(); gfxSwapBuffers(); From 73a2063707882113930d012a941f0ca4aa5ad305 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sun, 18 Mar 2018 19:57:32 +0000 Subject: [PATCH 21/72] Add more libs into the release zip --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index a4cc674..1ec557c 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -65,7 +65,7 @@ distfPY: linkPY cp $(PYDIR)/Lib/socket.py $(PYDIR)/Lib/socket.pyX cat $(PYDIR)/Lib/socket.pyX | sed 's/'"'"'getpeername'"'"', //g' >$(PYDIR)/Lib/socket.py rm $(PYDIR)/Lib/socket.pyX - cd $(PYDIR)/Lib && ls *.py -1 | xargs zip ../../$(OUTDIR)/python.zip && find json/ encodings/ -type f | xargs zip ../../$(OUTDIR)/python.zip + cd $(PYDIR)/Lib && ls *.py -1 | xargs zip ../../$(OUTDIR)/python.zip && find json/ encodings/ html/ http/ test/ urllib/ collections/ email/ sqlite3/ logging/ xml/ -type f | xargs zip ../../$(OUTDIR)/python.zip touch distfPY From c2f0f6c76451c26321cc8da37ff86ef6d98cc062 Mon Sep 17 00:00:00 2001 From: Aileen Lumina Date: Mon, 19 Mar 2018 11:51:43 +0100 Subject: [PATCH 22/72] Temp fix for DNS issues --- python_config/pyconfig.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index 41a9fc9..1463f6e 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -334,7 +334,7 @@ #define HAVE_GCC_UINT128_T 1 /* Define if you have the getaddrinfo function. */ -#define HAVE_GETADDRINFO 1 +#undef HAVE_GETADDRINFO // TODO clarify what these are #define SOCK_RAW 3 @@ -380,7 +380,7 @@ /* #define HAVE_GETLOGIN 1 */ // original: #define HAVE_GETLOGIN 1 /* Define to 1 if you have the `getnameinfo' function. */ -#define HAVE_GETNAMEINFO 1 +#undef HAVE_GETNAMEINFO /* Define if you have the 'getpagesize' function. */ #define HAVE_GETPAGESIZE 1 From 604dade7fcdd43a2438b0df8e50fc3adfe5fdefe Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 19 Mar 2018 15:26:21 +0000 Subject: [PATCH 23/72] Pretend we have lstat --- python_config/pyconfig.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index 1463f6e..51af129 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -573,7 +573,7 @@ #define HAVE_LONG_LONG 1 /* Define to 1 if you have the `lstat' function. */ -/* #define HAVE_LSTAT 1 */ // original: #define HAVE_LSTAT 1 +#define HAVE_LSTAT 1 /* We don't really have lstat, but sed replaces it with stat */ /* Define to 1 if you have the `lutimes' function. */ /* #define HAVE_LUTIMES 1 */ // original: #define HAVE_LUTIMES 1 From 6cb46291f4c25d57730ebc5ca4aa382bb9176a99 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 19 Mar 2018 16:51:29 +0000 Subject: [PATCH 24/72] Use patched copy of getaddrinfo.c --- python_build/Makefile | 1 + python_config/getaddrinfo.c | 638 ++++++++++++++++++++++++++++++++++++ 2 files changed, 639 insertions(+) create mode 100644 python_config/getaddrinfo.c diff --git a/python_build/Makefile b/python_build/Makefile index 1ec557c..c2dc7cf 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -78,6 +78,7 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/random.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Modules/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old diff --git a/python_config/getaddrinfo.c b/python_config/getaddrinfo.c new file mode 100644 index 0000000..d8167ea --- /dev/null +++ b/python_config/getaddrinfo.c @@ -0,0 +1,638 @@ +/* + * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * "#ifdef FAITH" part is local hack for supporting IPv4-v6 translator. + * + * Issues to be discussed: + * - Thread safe-ness must be checked. + * - Return values. There are nonstandard return values defined and used + * in the source code. This is because RFC2133 is silent about which error + * code must be returned for which situation. + * - PF_UNSPEC case would be handled in getipnodebyname() with the AI_ALL flag. + */ + +#if 0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "addrinfo.h" +#endif + +#if defined(__KAME__) && defined(ENABLE_IPV6) +# define FAITH +#endif + +#define SUCCESS 0 +#define GAI_ANY 0 +#define YES 1 +#define NO 0 + +#ifdef FAITH +static int translate = NO; +static struct in6_addr faith_prefix = IN6ADDR_GAI_ANY_INIT; +#endif + +static const char in_addrany[] = { 0, 0, 0, 0 }; +static const char in6_addrany[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; +static const char in_loopback[] = { 127, 0, 0, 1 }; +static const char in6_loopback[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 +}; + +struct sockinet { + u_char si_len; + u_char si_family; + u_short si_port; +}; + +static struct gai_afd { + int a_af; + int a_addrlen; + int a_socklen; + int a_off; + const char *a_addrany; + const char *a_loopback; +} gai_afdl [] = { +#ifdef ENABLE_IPV6 +#define N_INET6 0 + {PF_INET6, sizeof(struct in6_addr), + sizeof(struct sockaddr_in6), + offsetof(struct sockaddr_in6, sin6_addr), + in6_addrany, in6_loopback}, +#define N_INET 1 +#else +#define N_INET 0 +#endif + {PF_INET, sizeof(struct in_addr), + sizeof(struct sockaddr_in), + offsetof(struct sockaddr_in, sin_addr), + in_addrany, in_loopback}, + {0, 0, 0, 0, NULL, NULL}, +}; + +#ifdef ENABLE_IPV6 +#define PTON_MAX 16 +#else +#define PTON_MAX 4 +#endif + +#ifndef IN_MULTICAST +#define IN_MULTICAST(i) (((i) & 0xf0000000U) == 0xe0000000U) +#endif + +#ifndef IN_EXPERIMENTAL +#define IN_EXPERIMENTAL(i) (((i) & 0xe0000000U) == 0xe0000000U) +#endif + +#ifndef IN_LOOPBACKNET +#define IN_LOOPBACKNET 127 +#endif + +static int get_name(const char *, struct gai_afd *, + struct addrinfo **, char *, struct addrinfo *, + int); +static int get_addr(const char *, int, struct addrinfo **, + struct addrinfo *, int); +static int str_isnumber(const char *); + +static char *ai_errlist[] = { + "success.", + "address family for hostname not supported.", /* EAI_ADDRFAMILY */ + "temporary failure in name resolution.", /* EAI_AGAIN */ + "invalid value for ai_flags.", /* EAI_BADFLAGS */ + "non-recoverable failure in name resolution.", /* EAI_FAIL */ + "ai_family not supported.", /* EAI_FAMILY */ + "memory allocation failure.", /* EAI_MEMORY */ + "no address associated with hostname.", /* EAI_NODATA */ + "hostname nor servname provided, or not known.",/* EAI_NONAME */ + "servname not supported for ai_socktype.", /* EAI_SERVICE */ + "ai_socktype not supported.", /* EAI_SOCKTYPE */ + "system error returned in errno.", /* EAI_SYSTEM */ + "invalid value for hints.", /* EAI_BADHINTS */ + "resolved protocol is unknown.", /* EAI_PROTOCOL */ + "unknown error.", /* EAI_MAX */ +}; + +#define GET_CANONNAME(ai, str) \ +if (pai->ai_flags & AI_CANONNAME) {\ + if (((ai)->ai_canonname = (char *)malloc(strlen(str) + 1)) != NULL) {\ + strcpy((ai)->ai_canonname, (str));\ + } else {\ + error = EAI_MEMORY;\ + goto free;\ + }\ +} + +#ifdef HAVE_SOCKADDR_SA_LEN +#define GET_AI(ai, gai_afd, addr, port) {\ + char *p;\ + if (((ai) = (struct addrinfo *)malloc(sizeof(struct addrinfo) +\ + ((gai_afd)->a_socklen)))\ + == NULL) goto free;\ + memcpy(ai, pai, sizeof(struct addrinfo));\ + (ai)->ai_addr = (struct sockaddr *)((ai) + 1);\ + memset((ai)->ai_addr, 0, (gai_afd)->a_socklen);\ + (ai)->ai_addr->sa_len = (ai)->ai_addrlen = (gai_afd)->a_socklen;\ + (ai)->ai_addr->sa_family = (ai)->ai_family = (gai_afd)->a_af;\ + ((struct sockinet *)(ai)->ai_addr)->si_port = port;\ + p = (char *)((ai)->ai_addr);\ + memcpy(p + (gai_afd)->a_off, (addr), (gai_afd)->a_addrlen);\ +} +#else +#define GET_AI(ai, gai_afd, addr, port) {\ + char *p;\ + if (((ai) = (struct addrinfo *)malloc(sizeof(struct addrinfo) +\ + ((gai_afd)->a_socklen)))\ + == NULL) goto free;\ + memcpy(ai, pai, sizeof(struct addrinfo));\ + (ai)->ai_addr = (struct sockaddr *)((ai) + 1);\ + memset((ai)->ai_addr, 0, (gai_afd)->a_socklen);\ + (ai)->ai_addrlen = (gai_afd)->a_socklen;\ + (ai)->ai_addr->sa_family = (ai)->ai_family = (gai_afd)->a_af;\ + ((struct sockinet *)(ai)->ai_addr)->si_port = port;\ + p = (char *)((ai)->ai_addr);\ + memcpy(p + (gai_afd)->a_off, (addr), (gai_afd)->a_addrlen);\ +} +#endif + +#define ERR(err) { error = (err); goto bad; } + +char * +gai_strerror(int ecode) +{ + if (ecode < 0 || ecode > EAI_MAX) + ecode = EAI_MAX; + return ai_errlist[ecode]; +} + +void +freeaddrinfo(struct addrinfo *ai) +{ + struct addrinfo *next; + + do { + next = ai->ai_next; + if (ai->ai_canonname) + free(ai->ai_canonname); + /* no need to free(ai->ai_addr) */ + free(ai); + } while ((ai = next) != NULL); +} + +static int +str_isnumber(const char *p) +{ + unsigned char *q = (unsigned char *)p; + while (*q) { + if (! isdigit(*q)) + return NO; + q++; + } + return YES; +} + +int +getaddrinfo(const char*hostname, const char*servname, + const struct addrinfo *hints, struct addrinfo **res) +{ + struct addrinfo sentinel; + struct addrinfo *top = NULL; + struct addrinfo *cur; + int i, error = 0; + char pton[PTON_MAX]; + struct addrinfo ai; + struct addrinfo *pai; + u_short port; + +#ifdef FAITH + static int firsttime = 1; + + if (firsttime) { + /* translator hack */ + { + char *q = getenv("GAI"); + if (q && inet_pton(AF_INET6, q, &faith_prefix) == 1) + translate = YES; + } + firsttime = 0; + } +#endif + + /* initialize file static vars */ + sentinel.ai_next = NULL; + cur = &sentinel; + pai = &ai; + pai->ai_flags = 0; + pai->ai_family = PF_UNSPEC; + pai->ai_socktype = GAI_ANY; + pai->ai_protocol = GAI_ANY; + pai->ai_addrlen = 0; + pai->ai_canonname = NULL; + pai->ai_addr = NULL; + pai->ai_next = NULL; + port = GAI_ANY; + + if (hostname == NULL && servname == NULL) + return EAI_NONAME; + if (hints) { + /* error check for hints */ + if (hints->ai_addrlen || hints->ai_canonname || + hints->ai_addr || hints->ai_next) + ERR(EAI_BADHINTS); /* xxx */ + if (hints->ai_flags & ~AI_MASK) + ERR(EAI_BADFLAGS); + switch (hints->ai_family) { + case PF_UNSPEC: + case PF_INET: +#ifdef ENABLE_IPV6 + case PF_INET6: +#endif + break; + default: + ERR(EAI_FAMILY); + } + memcpy(pai, hints, sizeof(*pai)); + switch (pai->ai_socktype) { + case GAI_ANY: + switch (pai->ai_protocol) { + case GAI_ANY: + break; + case IPPROTO_UDP: + pai->ai_socktype = SOCK_DGRAM; + break; + case IPPROTO_TCP: + pai->ai_socktype = SOCK_STREAM; + break; + default: + pai->ai_socktype = SOCK_RAW; + break; + } + break; + case SOCK_RAW: + break; + case SOCK_DGRAM: + if (pai->ai_protocol != IPPROTO_UDP && + pai->ai_protocol != GAI_ANY) + ERR(EAI_BADHINTS); /*xxx*/ + pai->ai_protocol = IPPROTO_UDP; + break; + case SOCK_STREAM: + if (pai->ai_protocol != IPPROTO_TCP && + pai->ai_protocol != GAI_ANY) + ERR(EAI_BADHINTS); /*xxx*/ + pai->ai_protocol = IPPROTO_TCP; + break; + default: + ERR(EAI_SOCKTYPE); + /* unreachable */ + } + } + + /* + * service port + */ + if (servname) { + if (str_isnumber(servname)) { + if (pai->ai_socktype == GAI_ANY) { + /* caller accept *GAI_ANY* socktype */ + pai->ai_socktype = SOCK_DGRAM; + pai->ai_protocol = IPPROTO_UDP; + } + port = htons((u_short)atoi(servname)); + } else { + struct servent *sp; + char *proto; + + proto = NULL; + switch (pai->ai_socktype) { + case GAI_ANY: + proto = NULL; + break; + case SOCK_DGRAM: + proto = "udp"; + break; + case SOCK_STREAM: + proto = "tcp"; + break; + default: + fprintf(stderr, "panic!\n"); + break; + } + if ((sp = getservbyname(servname, proto)) == NULL) + ERR(EAI_SERVICE); + port = sp->s_port; + if (pai->ai_socktype == GAI_ANY) { + if (strcmp(sp->s_proto, "udp") == 0) { + pai->ai_socktype = SOCK_DGRAM; + pai->ai_protocol = IPPROTO_UDP; + } else if (strcmp(sp->s_proto, "tcp") == 0) { + pai->ai_socktype = SOCK_STREAM; + pai->ai_protocol = IPPROTO_TCP; + } else + ERR(EAI_PROTOCOL); /*xxx*/ + } + } + } + + /* + * hostname == NULL. + * passive socket -> anyaddr (0.0.0.0 or ::) + * non-passive socket -> localhost (127.0.0.1 or ::1) + */ + if (hostname == NULL) { + struct gai_afd *gai_afd; + + for (gai_afd = &gai_afdl[0]; gai_afd->a_af; gai_afd++) { + if (!(pai->ai_family == PF_UNSPEC + || pai->ai_family == gai_afd->a_af)) { + continue; + } + + if (pai->ai_flags & AI_PASSIVE) { + GET_AI(cur->ai_next, gai_afd, gai_afd->a_addrany, port); + /* xxx meaningless? + * GET_CANONNAME(cur->ai_next, "anyaddr"); + */ + } else { + GET_AI(cur->ai_next, gai_afd, gai_afd->a_loopback, + port); + /* xxx meaningless? + * GET_CANONNAME(cur->ai_next, "localhost"); + */ + } + cur = cur->ai_next; + } + top = sentinel.ai_next; + if (top) + goto good; + else + ERR(EAI_FAMILY); + } + + /* hostname as numeric name */ + for (i = 0; gai_afdl[i].a_af; i++) { + if (inet_pton(gai_afdl[i].a_af, hostname, pton)) { + u_long v4a; +#ifdef ENABLE_IPV6 + u_char pfx; +#endif + + switch (gai_afdl[i].a_af) { + case AF_INET: + v4a = ((struct in_addr *)pton)->s_addr; + v4a = ntohl(v4a); + if (IN_MULTICAST(v4a) || IN_EXPERIMENTAL(v4a)) + pai->ai_flags &= ~AI_CANONNAME; + v4a >>= IN_CLASSA_NSHIFT; + if (v4a == 0 || v4a == IN_LOOPBACKNET) + pai->ai_flags &= ~AI_CANONNAME; + break; +#ifdef ENABLE_IPV6 + case AF_INET6: + pfx = ((struct in6_addr *)pton)->s6_addr[0]; + if (pfx == 0 || pfx == 0xfe || pfx == 0xff) + pai->ai_flags &= ~AI_CANONNAME; + break; +#endif + } + + if (pai->ai_family == gai_afdl[i].a_af || + pai->ai_family == PF_UNSPEC) { + if (! (pai->ai_flags & AI_CANONNAME)) { + GET_AI(top, &gai_afdl[i], pton, port); + goto good; + } + /* + * if AI_CANONNAME and if reverse lookup + * fail, return ai anyway to pacify + * calling application. + * + * XXX getaddrinfo() is a name->address + * translation function, and it looks strange + * that we do addr->name translation here. + */ + get_name(pton, &gai_afdl[i], &top, pton, pai, port); + goto good; + } else + ERR(EAI_FAMILY); /*xxx*/ + } + } + + if (pai->ai_flags & AI_NUMERICHOST) + ERR(EAI_NONAME); + + /* hostname as alphabetical name */ + error = get_addr(hostname, pai->ai_family, &top, pai, port); + if (error == 0) { + if (top) { + good: + *res = top; + return SUCCESS; + } else + error = EAI_FAIL; + } + free: + if (top) + freeaddrinfo(top); + bad: + *res = NULL; + return error; +} + +static int +get_name(addr, gai_afd, res, numaddr, pai, port0) + const char *addr; + struct gai_afd *gai_afd; + struct addrinfo **res; + char *numaddr; + struct addrinfo *pai; + int port0; +{ + u_short port = port0 & 0xffff; + struct hostent *hp; + struct addrinfo *cur; + int error = 0; +#ifdef ENABLE_IPV6 + int h_error; +#endif + +#ifdef ENABLE_IPV6 + hp = getipnodebyaddr(addr, gai_afd->a_addrlen, gai_afd->a_af, &h_error); +#else + hp = gethostbyaddr(addr, gai_afd->a_addrlen, AF_INET); +#endif + if (hp && hp->h_name && hp->h_name[0] && hp->h_addr_list[0]) { + GET_AI(cur, gai_afd, hp->h_addr_list[0], port); + GET_CANONNAME(cur, hp->h_name); + } else + GET_AI(cur, gai_afd, numaddr, port); + +#ifdef ENABLE_IPV6 + if (hp) + freehostent(hp); +#endif + *res = cur; + return SUCCESS; + free: + if (cur) + freeaddrinfo(cur); +#ifdef ENABLE_IPV6 + if (hp) + freehostent(hp); +#endif + /* bad: */ + *res = NULL; + return error; +} + +static int +get_addr(hostname, af, res, pai, port0) + const char *hostname; + int af; + struct addrinfo **res; + struct addrinfo *pai; + int port0; +{ + u_short port = port0 & 0xffff; + struct addrinfo sentinel; + struct hostent *hp; + struct addrinfo *top, *cur; + struct gai_afd *gai_afd; + int i, error = 0, h_error; + char *ap; + + top = NULL; + sentinel.ai_next = NULL; + cur = &sentinel; +#ifdef ENABLE_IPV6 + if (af == AF_UNSPEC) { + hp = getipnodebyname(hostname, AF_INET6, + AI_ADDRCONFIG|AI_ALL|AI_V4MAPPED, &h_error); + } else + hp = getipnodebyname(hostname, af, AI_ADDRCONFIG, &h_error); +#else + hp = gethostbyname(hostname); + h_error = h_errno; +#endif + if (hp == NULL) { + switch (h_error) { + case HOST_NOT_FOUND: + case NO_DATA: + error = EAI_NODATA; + break; + case TRY_AGAIN: + error = EAI_AGAIN; + break; + case NO_RECOVERY: + default: + error = EAI_FAIL; + break; + } + goto free; + } + + if ((hp->h_name == NULL) || (hp->h_name[0] == 0) || + (hp->h_addr_list[0] == NULL)) { + error = EAI_FAIL; + goto free; + } + + for (i = 0; (ap = hp->h_addr_list[i]) != NULL; i++) { + switch (af) { +#ifdef ENABLE_IPV6 + case AF_INET6: + gai_afd = &gai_afdl[N_INET6]; + break; +#endif +#ifndef ENABLE_IPV6 + default: /* AF_UNSPEC */ +#endif + case AF_INET: + gai_afd = &gai_afdl[N_INET]; + break; +#ifdef ENABLE_IPV6 + default: /* AF_UNSPEC */ + if (IN6_IS_ADDR_V4MAPPED((struct in6_addr *)ap)) { + ap += sizeof(struct in6_addr) - + sizeof(struct in_addr); + gai_afd = &gai_afdl[N_INET]; + } else + gai_afd = &gai_afdl[N_INET6]; + break; +#endif + } +#ifdef FAITH + if (translate && gai_afd->a_af == AF_INET) { + struct in6_addr *in6; + + GET_AI(cur->ai_next, &gai_afdl[N_INET6], ap, port); + in6 = &((struct sockaddr_in6 *)cur->ai_next->ai_addr)->sin6_addr; + memcpy(&in6->s6_addr32[0], &faith_prefix, + sizeof(struct in6_addr) - sizeof(struct in_addr)); + memcpy(&in6->s6_addr32[3], ap, sizeof(struct in_addr)); + } else +#endif /* FAITH */ + GET_AI(cur->ai_next, gai_afd, ap, port); + if (cur == &sentinel) { + top = cur->ai_next; + GET_CANONNAME(top, hp->h_name); + } + cur = cur->ai_next; + } +#ifdef ENABLE_IPV6 + freehostent(hp); +#endif + *res = top; + return SUCCESS; + free: + if (top) + freeaddrinfo(top); +#ifdef ENABLE_IPV6 + if (hp) + freehostent(hp); +#endif +/* bad: */ + *res = NULL; + return error; +} From 6e29977b5d9d2ed4bedb303d86d6b438a028d255 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 19 Mar 2018 16:52:43 +0000 Subject: [PATCH 25/72] Dirty hack: Implement get_addr with inet_pton --- python_config/getaddrinfo.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python_config/getaddrinfo.c b/python_config/getaddrinfo.c index d8167ea..ed40977 100644 --- a/python_config/getaddrinfo.c +++ b/python_config/getaddrinfo.c @@ -545,6 +545,33 @@ get_addr(hostname, af, res, pai, port0) top = NULL; sentinel.ai_next = NULL; cur = &sentinel; + +#ifdef SWITCH // XXX dirty hack XXX + struct sockaddr_in *saddr; + + if (!(af == AF_INET || af == AF_UNSPEC)) + return EAI_NODATA; + + saddr = calloc(1, sizeof(*saddr)); + saddr->sin_family = AF_INET; + saddr->sin_port = port0; + + /* XXX libnx inet_pton is currently broken, we will ignore the return value for now... */ + if (inet_pton(AF_INET, hostname, &(saddr->sin_addr)) != 1 && 0) { + free(saddr); + error = EAI_NODATA; + goto free; + } + + top = calloc(1, sizeof(*top)); + top->ai_family = AF_INET; + top->ai_addrlen = sizeof(*saddr); + top->ai_addr = (struct sockaddr *) saddr; + + *res = top; + return SUCCESS; +#endif + #ifdef ENABLE_IPV6 if (af == AF_UNSPEC) { hp = getipnodebyname(hostname, AF_INET6, From c6251047c0f1fea056e95efc9e84f489f24f2287 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Tue, 20 Mar 2018 16:13:22 +0000 Subject: [PATCH 26/72] Fix makefile typo, add more modules to output dir --- python_build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index c2dc7cf..f582838 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -65,7 +65,7 @@ distfPY: linkPY cp $(PYDIR)/Lib/socket.py $(PYDIR)/Lib/socket.pyX cat $(PYDIR)/Lib/socket.pyX | sed 's/'"'"'getpeername'"'"', //g' >$(PYDIR)/Lib/socket.py rm $(PYDIR)/Lib/socket.pyX - cd $(PYDIR)/Lib && ls *.py -1 | xargs zip ../../$(OUTDIR)/python.zip && find json/ encodings/ html/ http/ test/ urllib/ collections/ email/ sqlite3/ logging/ xml/ -type f | xargs zip ../../$(OUTDIR)/python.zip + cd $(PYDIR)/Lib && ls *.py -1 | xargs zip ../../$(OUTDIR)/python.zip && find json/ encodings/ html/ http/ test/ urllib/ collections/ email/ sqlite3/ logging/ xml/ importlib/ asyncio/ -type f | xargs zip ../../$(OUTDIR)/python.zip touch distfPY @@ -78,7 +78,7 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/random.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Python/ - cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Modules/ + cp $(PYCONFIG_DIR)/getaddrinfo.c $(PYDIR)/Modules/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old From c8f4308b723e1589a8f22b6205f83c662e4dd653 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sat, 24 Mar 2018 17:30:50 +0000 Subject: [PATCH 27/72] Integrate _nx module --- python_build/Makefile | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index f582838..980248e 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -49,6 +49,8 @@ OUTDIR := nxpy$(PYVERS) PYDIR := Python-$(PYVERS) PYLINK := https://www.python.org/ftp/python/$(PYVERS)/Python-$(PYVERS).tgz PYFILE := py.tgz +_NXFILE := _nx.zip +_NX_VERSION := 174dfd099fcdb6dab8e521a2816e15f638de0436 .PHONY: all clean @@ -72,6 +74,11 @@ distfPY: linkPY linkPY: soospatchPY cd $(PYDIR) && make $(MAKEFLAGS) LIBRARY="$(ANAME)" LDLIBRARY="$(ANAME)" $(ANAME) && cd .. && cp $(PYDIR)/$(ANAME) . && touch linkPY +clone_NX: + wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NX_VERSION).zip" + unzip -o $(_NXFILE) + touch clone_NX + soospatchPY: compilePY cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ @@ -94,14 +101,15 @@ soospatchPY: compilePY compilePY: extractedPY patchPY cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-doc-strings --without-signal-module --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY -patchPY: +patchPY: clone_NX cp $(PYDIR)/configure $(PYDIR)/configure_old cat $(PYDIR)/configure_old | sed 's/ \*\-\*\-linux\*)/ \*\-\*\-linux\*\|aarch64\-none\-elf)/g' >$(PYDIR)/configure echo ac_cv_file__dev_ptmx=no >$(PYDIR)/config.site echo ac_cv_file__dev_ptc=no >>$(PYDIR)/config.site echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old - cat $(PYDIR)/Modules/Setup.dist_old | sed 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | sed 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | sed "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" >$(PYDIR)/Modules/Setup.dist + cat $(PYDIR)/Modules/Setup.dist_old | sed -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c' | sed 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | sed 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | sed "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" >$(PYDIR)/Modules/Setup.dist + cp -r _nx-*/_nx $(PYDIR)/Modules/ touch patchPY @@ -112,4 +120,4 @@ $(PYFILE): wget -O "$(PYFILE)" "$(PYLINK)" || curl -Lo "$(PYFILE)" "$(PYLINK)" clean: - @rm -rf $(PYDIR) $(PYFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY libpython*.a + @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY clone_NX _nx-* libpython*.a From ef81560c94656f42c2fb0be499c4fb497048c7b0 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Sat, 24 Mar 2018 18:17:43 +0000 Subject: [PATCH 28/72] save dumping, tcp repl, and mandelbrot (gfx) examples --- examples/_nx/mandelbrot.py | 34 +++++++++++++++++++++++++++++++++ examples/_nx/saves.py | 16 ++++++++++++++++ examples/tcp_repl.py | 39 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 examples/_nx/mandelbrot.py create mode 100644 examples/_nx/saves.py create mode 100644 examples/tcp_repl.py diff --git a/examples/_nx/mandelbrot.py b/examples/_nx/mandelbrot.py new file mode 100644 index 0000000..97e3a22 --- /dev/null +++ b/examples/_nx/mandelbrot.py @@ -0,0 +1,34 @@ +import _nx + +GfxMode_LinearDouble = 2 +WIDTH = 1280 +HEIGHT = 720 +MAX_ITER = 32 + +palette_r = [0, 8, 16, 24, 32, 40, 42, 38, 32, 22, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 57, 96, 137, 182, 216, 224, 232, 240, 248] +palette_g = [0, 1, 6, 13, 24, 37, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 110, 90, 66, 40, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] +palette_b = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 24, 45, 70, 97, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 202, 168, 130, 90, 46] + +def mandel(x0, y0): + x, y = (0.0, 0.0) + for i in range(MAX_ITER): + x, y = ( (x*x - y*y + x0), (2*x*y + y0) ) + if (x*x + y*y) >= 4: + return i + return 0 + +_nx.gfx_set_mode(GfxMode_LinearDouble) + +fb = bytearray([0]*WIDTH*HEIGHT*4) # RGBA8888 + +for y in range(HEIGHT): + y0 = (y/HEIGHT*2)-1 + for x in range(WIDTH): + x0 = (x/HEIGHT*2)-2.5 + iterations = mandel(x0, y0) + fb[(y*WIDTH+x)*4] = palette_r[iterations] + fb[(y*WIDTH+x)*4+1] = palette_g[iterations] + fb[(y*WIDTH+x)*4+2] = palette_b[iterations] + + _nx.gfx_set_framebuffer(fb) + _nx.gfx_flush_and_sync() diff --git a/examples/_nx/saves.py b/examples/_nx/saves.py new file mode 100644 index 0000000..15fd97b --- /dev/null +++ b/examples/_nx/saves.py @@ -0,0 +1,16 @@ +import _nx +import os +import sys + +title_id = 0x01007ef00011e000 # BotW + +_nx.account_initialize() +user_id = _nx.account_get_active_user() + +if user_id is None: + print("No active user, you need to launch and close a game prior to launching hbl.") + sys.exit() + +_nx.fs_mount_savedata("save", title_id, user_id) + +print(os.listdir("save:/")) diff --git a/examples/tcp_repl.py b/examples/tcp_repl.py new file mode 100644 index 0000000..8d21e43 --- /dev/null +++ b/examples/tcp_repl.py @@ -0,0 +1,39 @@ +import socket +import sys +import code +import time + +DEBUG_PORT = 1337 + +def remote_repl(local): + sys.stdout.flush() + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("0.0.0.0", DEBUG_PORT)) + s.listen(1) + + print("Waiting for remote debug connection on port {}.".format(DEBUG_PORT)) + + conn, addr = s.accept() + + print("Accepted remote debug connection from '{}'.".format(addr[0])) + + sys.stdout.flush() + sys.stderr.flush() + stdin_bak, stdout_bak, stderr_bak = sys.stdin, sys.stdout, sys.stderr + rfile = conn.makefile("r") + wfile = conn.makefile("w") + sys.stdin, sys.stdout, sys.stderr = rfile, wfile, wfile + + try: + code.interact(local=local) + except SystemExit: + pass + + conn.close() + s.close() + sys.stdin, sys.stdout, sys.stderr = stdin_bak, stdout_bak, stderr_bak + print("Remote debug session closed.") + +while 1: + remote_repl(locals()) + time.sleep(1) From 3cad67c923ea6689af7640b7ef4910db70c23218 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 26 Mar 2018 00:04:54 +0100 Subject: [PATCH 29/72] tabs -> spaces --- examples/_nx/mandelbrot.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/_nx/mandelbrot.py b/examples/_nx/mandelbrot.py index 97e3a22..a83c82f 100644 --- a/examples/_nx/mandelbrot.py +++ b/examples/_nx/mandelbrot.py @@ -10,25 +10,25 @@ palette_b = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 24, 45, 70, 97, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 202, 168, 130, 90, 46] def mandel(x0, y0): - x, y = (0.0, 0.0) - for i in range(MAX_ITER): - x, y = ( (x*x - y*y + x0), (2*x*y + y0) ) - if (x*x + y*y) >= 4: - return i - return 0 + x, y = (0.0, 0.0) + for i in range(MAX_ITER): + x, y = ( (x*x - y*y + x0), (2*x*y + y0) ) + if (x*x + y*y) >= 4: + return i + return 0 _nx.gfx_set_mode(GfxMode_LinearDouble) fb = bytearray([0]*WIDTH*HEIGHT*4) # RGBA8888 for y in range(HEIGHT): - y0 = (y/HEIGHT*2)-1 - for x in range(WIDTH): - x0 = (x/HEIGHT*2)-2.5 - iterations = mandel(x0, y0) - fb[(y*WIDTH+x)*4] = palette_r[iterations] - fb[(y*WIDTH+x)*4+1] = palette_g[iterations] - fb[(y*WIDTH+x)*4+2] = palette_b[iterations] - - _nx.gfx_set_framebuffer(fb) - _nx.gfx_flush_and_sync() + y0 = (y/HEIGHT*2)-1 + for x in range(WIDTH): + x0 = (x/HEIGHT*2)-2.5 + iterations = mandel(x0, y0) + fb[(y*WIDTH+x)*4] = palette_r[iterations] + fb[(y*WIDTH+x)*4+1] = palette_g[iterations] + fb[(y*WIDTH+x)*4+2] = palette_b[iterations] + + _nx.gfx_set_framebuffer(fb) + _nx.gfx_flush_and_sync() From 724ae31a9eb8144d0d0346727add5fb895d2bb40 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 26 Mar 2018 10:04:50 +0100 Subject: [PATCH 30/72] Integrate nx package --- Makefile | 1 + python_build/Makefile | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d0f1b55..23c7b7f 100644 --- a/Makefile +++ b/Makefile @@ -152,6 +152,7 @@ cpython: dist: $(BUILD) mkdir -p $(LIBDIR) unzip $(PY_BUILD)/nxpy3.5.3/python.zip -d $(LIBDIR) + cp -r $(PY_BUILD)/nx-*/nx $(LIBDIR) cp $(OUTPUT).{nro,nacp} $(DIST_DIR)/ cp examples/hello.py $(DIST_DIR)/main.py cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) diff --git a/python_build/Makefile b/python_build/Makefile index 980248e..5874599 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -51,6 +51,8 @@ PYLINK := https://www.python.org/ftp/python/$(PYVERS)/Python-$(PYVERS).tgz PYFILE := py.tgz _NXFILE := _nx.zip _NX_VERSION := 174dfd099fcdb6dab8e521a2816e15f638de0436 +NXPKGFILE := nxpkg.zip +NXPKG_VERSION := fe22f98eed557c0efadd2a52d2b0542ef1d246c1 .PHONY: all clean @@ -76,7 +78,9 @@ linkPY: soospatchPY clone_NX: wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NX_VERSION).zip" + wget -O $(NXPKGFILE) "https://github.com/nx-python/nx/archive/$(NXPKG_VERSION).zip" unzip -o $(_NXFILE) + unzip -o $(NXPKGFILE) touch clone_NX soospatchPY: compilePY @@ -120,4 +124,4 @@ $(PYFILE): wget -O "$(PYFILE)" "$(PYLINK)" || curl -Lo "$(PYFILE)" "$(PYLINK)" clean: - @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY clone_NX _nx-* libpython*.a + @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXPKGFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY clone_NX _nx-* nx-* libpython*.a From dff80f420e20d248e4535297764729318ad9c466 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Fri, 30 Mar 2018 08:03:01 +0200 Subject: [PATCH 31/72] Update to latest nx, Pynx -> PyNX --- .gitignore | 2 +- LICENSE.md | 2 +- Makefile | 15 ++++++++------- README.md | 14 +++++++------- python_build/Makefile | 20 ++++++++++---------- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 75949ee..974c849 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ python_build/* !python_build/Makefile build/ -Pynx.* +PyNX.* diff --git a/LICENSE.md b/LICENSE.md index 6c56934..bade95b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright 2018 Pynx Authors +Copyright 2018 PyNX Authors Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. diff --git a/Makefile b/Makefile index 23c7b7f..64a9bf7 100644 --- a/Makefile +++ b/Makefile @@ -31,16 +31,16 @@ include $(DEVKITPRO)/libnx/switch_rules #--------------------------------------------------------------------------------- TARGET := $(notdir $(CURDIR)) BUILD := build -PY_BUILD := python_build -DIST_DIR := $(BUILD)/$(TARGET) -LIBDIR := $(DIST_DIR)/lib/python3.5 +PY_BUILD := python_build +DIST_DIR := $(BUILD)/$(TARGET) +LIBDIR := $(DIST_DIR)/lib/python3.5 SOURCES := source DATA := data INCLUDES := include EXEFS_SRC := exefs_src -APP_TITLEID := Pynx -APP_AUTHOR := nx-python Authors, Python Software Foundation -APP_VERSION := 0.2.0-alpha +APP_TITLE := PyNX +APP_AUTHOR := nx-python, Python Software Foundation +APP_VERSION := 0.3.0-alpha #--------------------------------------------------------------------------------- # options for code generation @@ -153,7 +153,8 @@ dist: $(BUILD) mkdir -p $(LIBDIR) unzip $(PY_BUILD)/nxpy3.5.3/python.zip -d $(LIBDIR) cp -r $(PY_BUILD)/nx-*/nx $(LIBDIR) - cp $(OUTPUT).{nro,nacp} $(DIST_DIR)/ + cp $(OUTPUT).nro $(DIST_DIR)/ + cp $(OUTPUT).nacp $(DIST_DIR)/ cp examples/hello.py $(DIST_DIR)/main.py cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) diff --git a/README.md b/README.md index cf7de41..f2fc304 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -# Pynx +# PyNX -**nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **Pynx** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `pynx.nro`, and it will be executed as soon as you launch Pynx from the Homebrew Menu. Currently, **only Python 2.7 is supported**. +**nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **PyNX** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `PyNX.nro`, and it will be executed as soon as you launch PyNX from the Homebrew Menu. Currently, **only Python 2.7 is supported**. -## Running Pynx on your Switch +## Running PyNX on your Switch -You don't have to compile Pynx, you can [just grab a release build](https://github.com/nx-python/Pynx/releases) and copy the content of the ZIP archive into the `/switch` folder on your SD card. Pynx will appear on the Homebrew Menu. +You don't have to compile PyNX, you can [just grab a release build](https://github.com/nx-python/PyNX/releases) and copy the content of the ZIP archive into the `/switch` folder on your SD card. PyNX will appear on the Homebrew Menu. -## Compiling Pynx +## Compiling PyNX -Compile Pynx using `make`. This will create a `Pynx` directory and build everything in there. Compiling might take a while, grab a coffee or whatever in the meantime if you like. Afterwards, create a distributable version using `make dist`. It will appear in the `build` directory. +Compile PyNX using `make`. This will create a `PyNX` directory and build everything in there. Compiling might take a while, grab a coffee or whatever in the meantime if you like. Afterwards, create a distributable version using `make dist`. It will appear in the `build` directory. ## Having an issue? -If you encounter a problem, make sure to [join our Discord](https://discord.gg/5Ga2Whf) and tell us about it, or, alternatively, you can use GitHub's [issue tracker](https://github.com/nx-python/Pynx/issues) to report an issue. +If you encounter a problem, make sure to [join our Discord](https://discord.gg/5Ga2Whf) and tell us about it, or, alternatively, you can use GitHub's [issue tracker](https://github.com/nx-python/PyNX/issues) to report an issue. diff --git a/python_build/Makefile b/python_build/Makefile index 5874599..3922ced 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -50,9 +50,9 @@ PYDIR := Python-$(PYVERS) PYLINK := https://www.python.org/ftp/python/$(PYVERS)/Python-$(PYVERS).tgz PYFILE := py.tgz _NXFILE := _nx.zip -_NX_VERSION := 174dfd099fcdb6dab8e521a2816e15f638de0436 -NXPKGFILE := nxpkg.zip -NXPKG_VERSION := fe22f98eed557c0efadd2a52d2b0542ef1d246c1 +_NXVERS := 174dfd099fcdb6dab8e521a2816e15f638de0436 +NXFILE := nx.zip +NXVERS := c1069b999d54bf72abce8d993479ea8dc53af45b .PHONY: all clean @@ -76,12 +76,12 @@ distfPY: linkPY linkPY: soospatchPY cd $(PYDIR) && make $(MAKEFLAGS) LIBRARY="$(ANAME)" LDLIBRARY="$(ANAME)" $(ANAME) && cd .. && cp $(PYDIR)/$(ANAME) . && touch linkPY -clone_NX: - wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NX_VERSION).zip" - wget -O $(NXPKGFILE) "https://github.com/nx-python/nx/archive/$(NXPKG_VERSION).zip" +cloneNX: + wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NXVERS).zip" + wget -O $(NXFILE) "https://github.com/nx-python/nx/archive/$(NXVERS).zip" unzip -o $(_NXFILE) - unzip -o $(NXPKGFILE) - touch clone_NX + unzip -o $(NXFILE) + touch cloneNX soospatchPY: compilePY cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ @@ -105,7 +105,7 @@ soospatchPY: compilePY compilePY: extractedPY patchPY cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-doc-strings --without-signal-module --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY -patchPY: clone_NX +patchPY: cloneNX cp $(PYDIR)/configure $(PYDIR)/configure_old cat $(PYDIR)/configure_old | sed 's/ \*\-\*\-linux\*)/ \*\-\*\-linux\*\|aarch64\-none\-elf)/g' >$(PYDIR)/configure echo ac_cv_file__dev_ptmx=no >$(PYDIR)/config.site @@ -124,4 +124,4 @@ $(PYFILE): wget -O "$(PYFILE)" "$(PYLINK)" || curl -Lo "$(PYFILE)" "$(PYLINK)" clean: - @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXPKGFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY clone_NX _nx-* nx-* libpython*.a + @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY cloneNX _nx-* nx-* libpython*.a From 7ef821e6ea65ed97e557682ab0b72260f0f64dbd Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Fri, 30 Mar 2018 21:06:14 +0200 Subject: [PATCH 32/72] Update to latest nx --- Makefile | 2 +- python_build/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 64a9bf7..40eb575 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := PyNX APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.3.0-alpha +APP_VERSION := 0.3.1-alpha #--------------------------------------------------------------------------------- # options for code generation diff --git a/python_build/Makefile b/python_build/Makefile index 3922ced..858ce40 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -52,7 +52,7 @@ PYFILE := py.tgz _NXFILE := _nx.zip _NXVERS := 174dfd099fcdb6dab8e521a2816e15f638de0436 NXFILE := nx.zip -NXVERS := c1069b999d54bf72abce8d993479ea8dc53af45b +NXVERS := 4d3e57854f6adc6848af222f9f7582e30c7cf6b1 .PHONY: all clean From d5c5a85fe847ab437b08c5ad9754f13692a44ab4 Mon Sep 17 00:00:00 2001 From: tumGER <25822956+tumGER@users.noreply.github.com> Date: Thu, 19 Apr 2018 22:47:55 +0200 Subject: [PATCH 33/72] Change the readme for the real python version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f2fc304..3a5d8a0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PyNX -**nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **PyNX** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `PyNX.nro`, and it will be executed as soon as you launch PyNX from the Homebrew Menu. Currently, **only Python 2.7 is supported**. +**nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **PyNX** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `PyNX.nro`, and it will be executed as soon as you launch PyNX from the Homebrew Menu. Currently, **Python 3.5 is supported**. ## Running PyNX on your Switch From 95feeb341185c3fb9b0eed8f34804164c9dc47cb Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Thu, 10 May 2018 23:51:49 +0200 Subject: [PATCH 34/72] Update _nx and nx --- Makefile | 2 +- README.md | 2 +- python_build/Makefile | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 40eb575..7a0046f 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := PyNX APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.3.1-alpha +APP_VERSION := 0.3.2-alpha #--------------------------------------------------------------------------------- # options for code generation diff --git a/README.md b/README.md index f2fc304..3a5d8a0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PyNX -**nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **PyNX** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `PyNX.nro`, and it will be executed as soon as you launch PyNX from the Homebrew Menu. Currently, **only Python 2.7 is supported**. +**nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **PyNX** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `PyNX.nro`, and it will be executed as soon as you launch PyNX from the Homebrew Menu. Currently, **Python 3.5 is supported**. ## Running PyNX on your Switch diff --git a/python_build/Makefile b/python_build/Makefile index 858ce40..e698438 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -50,9 +50,9 @@ PYDIR := Python-$(PYVERS) PYLINK := https://www.python.org/ftp/python/$(PYVERS)/Python-$(PYVERS).tgz PYFILE := py.tgz _NXFILE := _nx.zip -_NXVERS := 174dfd099fcdb6dab8e521a2816e15f638de0436 +_NXVERS := c4eebe97a4e7ba952ebffd7012b34a768ae96dd4 NXFILE := nx.zip -NXVERS := 4d3e57854f6adc6848af222f9f7582e30c7cf6b1 +NXVERS := a969b53211e745c3697e700524f1c54183e7f53f .PHONY: all clean From 3ce5462a701348b56ec15aca6e39e1d3f294df58 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Tue, 15 May 2018 20:24:13 +0200 Subject: [PATCH 35/72] Updated _nx, nx, README --- Makefile | 2 +- README.md | 6 +++++- python_build/Makefile | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 7a0046f..c700012 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := PyNX APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.3.2-alpha +APP_VERSION := 0.4.0-alpha #--------------------------------------------------------------------------------- # options for code generation diff --git a/README.md b/README.md index 3a5d8a0..bf4c5fc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,11 @@ You don't have to compile PyNX, you can [just grab a release build](https://gith ## Compiling PyNX -Compile PyNX using `make`. This will create a `PyNX` directory and build everything in there. Compiling might take a while, grab a coffee or whatever in the meantime if you like. Afterwards, create a distributable version using `make dist`. It will appear in the `build` directory. +Compile PyNX using `make`. This will create a `build` directory and build everything in there. Compiling might take a while, grab a coffee or whatever in the meantime if you like. Afterwards, create a distributable version using `make dist`. It will appear in the `build` directory. + +## Documentation + +Documentation can be found on [ReadTheDocs](https://nx-python.readthedocs.io/en/latest/). ## Having an issue? diff --git a/python_build/Makefile b/python_build/Makefile index e698438..579202b 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -50,9 +50,9 @@ PYDIR := Python-$(PYVERS) PYLINK := https://www.python.org/ftp/python/$(PYVERS)/Python-$(PYVERS).tgz PYFILE := py.tgz _NXFILE := _nx.zip -_NXVERS := c4eebe97a4e7ba952ebffd7012b34a768ae96dd4 +_NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip -NXVERS := a969b53211e745c3697e700524f1c54183e7f53f +NXVERS := 4b8d31bf99d4971f9d59402dfbe47fb33d5978dc .PHONY: all clean From 32b68dde92219402e3989a96a6eaba3861c71d82 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Tue, 15 May 2018 20:47:16 +0200 Subject: [PATCH 36/72] Update nx again --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 579202b..215189e 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -52,7 +52,7 @@ PYFILE := py.tgz _NXFILE := _nx.zip _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip -NXVERS := 4b8d31bf99d4971f9d59402dfbe47fb33d5978dc +NXVERS := d20e94403bcc81c23b2997fd29aedc6ee81c0648 .PHONY: all clean From 8b7c0ee526d5326112ed54d3e72f4a9d682c4965 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Wed, 16 May 2018 04:15:53 +0100 Subject: [PATCH 37/72] Un-disable docstrings --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 215189e..b444046 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -103,7 +103,7 @@ soospatchPY: compilePY touch soospatchPY compilePY: extractedPY patchPY - cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-doc-strings --without-signal-module --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY + cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-signal-module --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY patchPY: cloneNX cp $(PYDIR)/configure $(PYDIR)/configure_old From ada05686e235c4162faa12000649b39184116666 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Wed, 16 May 2018 04:25:13 +0100 Subject: [PATCH 38/72] Calculate absolute import path --- source/main.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/main.c b/source/main.c index 0836cf8..b514c1b 100644 --- a/source/main.c +++ b/source/main.c @@ -31,8 +31,13 @@ int main(int argc, char *argv[]) printf("Python %s on %s\n", Py_GetVersion(), Py_GetPlatform()); /* set up import path */ + char cwd[PATH_MAX]; + getcwd(cwd, sizeof(cwd)); + /* Strip the leading sdmc: to workaround a bug somewhere... */ + char *stripped_cwd = strchr(cwd, '/'); + if (stripped_cwd == NULL) stripped_cwd = cwd; PyObject *sysPath = PySys_GetObject("path"); - PyObject *path = PyUnicode_FromString("./"); + PyObject *path = PyUnicode_FromString(cwd); PyList_Insert(sysPath, 0, path); FILE * mainpy = fopen(MAINPY, "r"); From 0f3f9d3da05315a5dd8b5d6e57688eb3a84d12c0 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Thu, 17 May 2018 03:38:33 +0100 Subject: [PATCH 39/72] ipc.h moved in libnx 1.2.0 --- python_config/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_config/random.c b/python_config/random.c index bd1107f..768489e 100644 --- a/python_config/random.c +++ b/python_config/random.c @@ -20,7 +20,7 @@ #ifdef SWITCH #include "switch/types.h" #include "switch/result.h" -#include "switch/ipc.h" +#include "switch/kernel/ipc.h" #include "switch/services/sm.h" #endif From 56f4b426b535e0448d7913b78b8af73d6a231fa3 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Fri, 18 May 2018 11:42:19 +0000 Subject: [PATCH 40/72] Bump version to v0.4.1-alpha --- Makefile | 2 +- python_build/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c700012..f314deb 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := PyNX APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.4.0-alpha +APP_VERSION := 0.4.1-alpha #--------------------------------------------------------------------------------- # options for code generation diff --git a/python_build/Makefile b/python_build/Makefile index b444046..d28205f 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -52,7 +52,7 @@ PYFILE := py.tgz _NXFILE := _nx.zip _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip -NXVERS := d20e94403bcc81c23b2997fd29aedc6ee81c0648 +NXVERS := 9b9d2dc61d6b0ecf9839ae470a26d5089d1102c1 .PHONY: all clean From 3516c5cd3d3bfe42971ec621f8026b4778276590 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Thu, 24 May 2018 04:39:51 +0200 Subject: [PATCH 41/72] Add app loader as new default main.py --- Makefile | 1 - examples/hello.py | 25 ----------------- examples/tcp_repl.py | 24 ++++++---------- main.py | 67 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 41 deletions(-) delete mode 100644 examples/hello.py create mode 100644 main.py diff --git a/Makefile b/Makefile index f314deb..a876584 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,6 @@ dist: $(BUILD) cp -r $(PY_BUILD)/nx-*/nx $(LIBDIR) cp $(OUTPUT).nro $(DIST_DIR)/ cp $(OUTPUT).nacp $(DIST_DIR)/ - cp examples/hello.py $(DIST_DIR)/main.py cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) #--------------------------------------------------------------------------------- diff --git a/examples/hello.py b/examples/hello.py deleted file mode 100644 index 8c71d59..0000000 --- a/examples/hello.py +++ /dev/null @@ -1,25 +0,0 @@ -print(""" - _ _ _ -| | | | | | -| | | | ___| | ___ ___ _ __ ___ ___ -| |/\\| |/ _ | |/ __/ _ \\| '_ ` _ \\ / _ \\ -\\ /\\ | __| | (_| (_) | | | | | | __/ - \\/ \\/ \\___|_|\\___\\___/|_| |_| |_|\\___| - - _ - | | - | |_ ___ - | __/ _ \\ - | || (_) | - \\__\\___/ - - ______ _ - | ___ \\ | | - | |_/ _ _ _ __ __ _| | - | __| | | | '_ \\\\ \\/ | | - | | | |_| | | | |> <|_| - \\_| \\__, |_| |_/_/\\_(_) - __/ | - |___/ - -Replace this file (main.py) with your own to get started. """) diff --git a/examples/tcp_repl.py b/examples/tcp_repl.py index 8d21e43..0f499ac 100644 --- a/examples/tcp_repl.py +++ b/examples/tcp_repl.py @@ -1,19 +1,18 @@ import socket import sys import code -import time + DEBUG_PORT = 1337 -def remote_repl(local): - sys.stdout.flush() - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.bind(("0.0.0.0", DEBUG_PORT)) - s.listen(1) - print("Waiting for remote debug connection on port {}.".format(DEBUG_PORT)) +if __name__ == '__main__': + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("0.0.0.0", DEBUG_PORT)) + sock.listen(1) + print("Waiting for remote debug connection on port {}.".format(DEBUG_PORT), flush=True) - conn, addr = s.accept() + conn, addr = sock.accept() print("Accepted remote debug connection from '{}'.".format(addr[0])) @@ -25,15 +24,10 @@ def remote_repl(local): sys.stdin, sys.stdout, sys.stderr = rfile, wfile, wfile try: - code.interact(local=local) + code.interact(local=locals()) except SystemExit: pass conn.close() - s.close() + sock.close() sys.stdin, sys.stdout, sys.stderr = stdin_bak, stdout_bak, stderr_bak - print("Remote debug session closed.") - -while 1: - remote_repl(locals()) - time.sleep(1) diff --git a/main.py b/main.py new file mode 100644 index 0000000..e937f43 --- /dev/null +++ b/main.py @@ -0,0 +1,67 @@ +import sys +from glob import glob +import os +import runpy +import time +import _nx +import nx +from nx.utils import AnsiMenu + + +sys.argv = [""] # workaround needed for runpy + +TILED_DOUBLE = 1 +MAIN_PY = 'main.py' +PYNX_DIR_PATH = os.getcwd() + + +def run_python_module(path: str): + runpy.run_path(path, run_name='__main__') + _nx.gfx_set_mode(TILED_DOUBLE) + print("{} exited.".format(selected), flush=True) + + +def clear_screen(): + sys.stdout.buffer.write(b"\x1b[2J") + sys.stdout.buffer.flush() + + +def confirmation_prompt(program_name: str): + name = program_name[:-3] if program_name.endswith('.py') else program_name + name = name[:-1] if name.endswith(os.sep) else name + print("Do you want to execute the Python program {}?\n" + "Press A to confirm, B to cancel.".format(name), flush=True) + while True: + nx.refresh_inputs() + if nx.p1.a_button.is_pressed: + return True + elif nx.p1.b_button.is_pressed: + return False + time.sleep(0.01) + + +if __name__ == '__main__': + while True: + clear_screen() + dirs = ["../"] + glob("*/") + scripts = glob("*.py") + if os.getcwd() == PYNX_DIR_PATH and MAIN_PY in scripts: + scripts.remove(MAIN_PY) + listing = dirs + scripts + file_menu = AnsiMenu(listing) + print("Listing for {}:".format(os.getcwd()), flush=True) + selected = listing[file_menu.query()] + clear_screen() + if selected in dirs: + os.chdir(selected) + if os.path.isfile(MAIN_PY) and not selected == '../' and not os.getcwd() == PYNX_DIR_PATH: + response = confirmation_prompt(selected) + clear_screen() + if response is True: + run_python_module(MAIN_PY) + os.chdir(os.pardir) + else: + response = confirmation_prompt(selected) + clear_screen() + if response is True: + run_python_module(selected) From 6337a4ab6aec7048282044254907bbd419e5a856 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Sat, 26 May 2018 22:32:25 +0000 Subject: [PATCH 42/72] Update main.py to use nx.utils.clear_terminal --- main.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/main.py b/main.py index e937f43..cdecf5c 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ import time import _nx import nx -from nx.utils import AnsiMenu +from nx.utils import AnsiMenu, clear_terminal sys.argv = [""] # workaround needed for runpy @@ -18,19 +18,14 @@ def run_python_module(path: str): runpy.run_path(path, run_name='__main__') _nx.gfx_set_mode(TILED_DOUBLE) - print("{} exited.".format(selected), flush=True) - - -def clear_screen(): - sys.stdout.buffer.write(b"\x1b[2J") - sys.stdout.buffer.flush() + print("{} exited.".format(selected)) def confirmation_prompt(program_name: str): name = program_name[:-3] if program_name.endswith('.py') else program_name name = name[:-1] if name.endswith(os.sep) else name print("Do you want to execute the Python program {}?\n" - "Press A to confirm, B to cancel.".format(name), flush=True) + "Press A to confirm, B to cancel.".format(name)) while True: nx.refresh_inputs() if nx.p1.a_button.is_pressed: @@ -42,26 +37,29 @@ def confirmation_prompt(program_name: str): if __name__ == '__main__': while True: - clear_screen() + clear_terminal() dirs = ["../"] + glob("*/") scripts = glob("*.py") if os.getcwd() == PYNX_DIR_PATH and MAIN_PY in scripts: scripts.remove(MAIN_PY) + for _dir in dirs: + if os.path.isfile(os.path.join(_dir, MAIN_PY)): + dirs.replace(_dir, _dir[:-1]) listing = dirs + scripts file_menu = AnsiMenu(listing) - print("Listing for {}:".format(os.getcwd()), flush=True) + print("Listing for {}:".format(os.getcwd())) selected = listing[file_menu.query()] - clear_screen() + clear_terminal() if selected in dirs: os.chdir(selected) if os.path.isfile(MAIN_PY) and not selected == '../' and not os.getcwd() == PYNX_DIR_PATH: response = confirmation_prompt(selected) - clear_screen() + clear_terminal() if response is True: run_python_module(MAIN_PY) os.chdir(os.pardir) else: response = confirmation_prompt(selected) - clear_screen() + clear_terminal() if response is True: run_python_module(selected) From 656c896375cc3f3cb7eb3851f5c5ce0c9281eff4 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Sat, 26 May 2018 22:56:57 +0000 Subject: [PATCH 43/72] Bump version to v0.5.0-alpha --- Makefile | 7 ++++--- python_build/Makefile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a876584..d7ef451 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ include $(DEVKITPRO)/libnx/switch_rules # - icon.jpg # - /default_icon.jpg #--------------------------------------------------------------------------------- -TARGET := $(notdir $(CURDIR)) +TARGET := PyNX BUILD := build PY_BUILD := python_build DIST_DIR := $(BUILD)/$(TARGET) @@ -38,9 +38,9 @@ SOURCES := source DATA := data INCLUDES := include EXEFS_SRC := exefs_src -APP_TITLE := PyNX +APP_TITLE := $(TARGET) APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.4.1-alpha +APP_VERSION := 0.5.0-alpha #--------------------------------------------------------------------------------- # options for code generation @@ -155,6 +155,7 @@ dist: $(BUILD) cp -r $(PY_BUILD)/nx-*/nx $(LIBDIR) cp $(OUTPUT).nro $(DIST_DIR)/ cp $(OUTPUT).nacp $(DIST_DIR)/ + cp main.py $(DIST_DIR)/ cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) #--------------------------------------------------------------------------------- diff --git a/python_build/Makefile b/python_build/Makefile index d28205f..efedfe0 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -52,7 +52,7 @@ PYFILE := py.tgz _NXFILE := _nx.zip _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip -NXVERS := 9b9d2dc61d6b0ecf9839ae470a26d5089d1102c1 +NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 .PHONY: all clean From f5d8857bd492e93728a6fca220e6c88830c73688 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Sat, 26 May 2018 23:05:08 +0000 Subject: [PATCH 44/72] Make loader start at /switch/ --- main.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index cdecf5c..67843ea 100644 --- a/main.py +++ b/main.py @@ -36,10 +36,12 @@ def confirmation_prompt(program_name: str): if __name__ == '__main__': + os.chdir(os.pardir) + while True: clear_terminal() - dirs = ["../"] + glob("*/") - scripts = glob("*.py") + dirs = [os.pardir + '/'] + glob('*/') + scripts = glob('*.py') if os.getcwd() == PYNX_DIR_PATH and MAIN_PY in scripts: scripts.remove(MAIN_PY) for _dir in dirs: @@ -52,7 +54,7 @@ def confirmation_prompt(program_name: str): clear_terminal() if selected in dirs: os.chdir(selected) - if os.path.isfile(MAIN_PY) and not selected == '../' and not os.getcwd() == PYNX_DIR_PATH: + if os.path.isfile(MAIN_PY) and not selected == os.pardir + '/' and not os.getcwd() == PYNX_DIR_PATH: response = confirmation_prompt(selected) clear_terminal() if response is True: From 2468c46ffc1e4e7cbe769d24b78b7f5c4227b77a Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Sun, 27 May 2018 15:05:48 +0000 Subject: [PATCH 45/72] Fix typo in main.py --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 67843ea..6138e89 100644 --- a/main.py +++ b/main.py @@ -46,7 +46,7 @@ def confirmation_prompt(program_name: str): scripts.remove(MAIN_PY) for _dir in dirs: if os.path.isfile(os.path.join(_dir, MAIN_PY)): - dirs.replace(_dir, _dir[:-1]) + dirs[dirs.index(_dir)] = _dir[:-1] listing = dirs + scripts file_menu = AnsiMenu(listing) print("Listing for {}:".format(os.getcwd())) From f8fe46d0e3e99ca1c06466154bc4cd8cad65fe83 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 28 May 2018 16:47:40 +0100 Subject: [PATCH 46/72] Actually calculate absolute home dir --- source/main.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/source/main.c b/source/main.c index b514c1b..0edbc1e 100644 --- a/source/main.c +++ b/source/main.c @@ -23,21 +23,23 @@ int main(int argc, char *argv[]) Py_NoUserSiteDirectory = 1; //Py_VerboseFlag += 1; - Py_SetPythonHome(L"./"); + /* Calculate absolute home dir */ + char cwd[PATH_MAX]; + getcwd(cwd, sizeof(cwd)); + /* Strip the leading sdmc: to workaround a bug somewhere... */ + char *stripped_cwd = strchr(cwd, '/'); + if (stripped_cwd == NULL) stripped_cwd = cwd; + + Py_SetPythonHome(Py_DecodeLocale(stripped_cwd, NULL)); Py_Initialize(); - + /* Print some info */ printf("Python %s on %s\n", Py_GetVersion(), Py_GetPlatform()); /* set up import path */ - char cwd[PATH_MAX]; - getcwd(cwd, sizeof(cwd)); - /* Strip the leading sdmc: to workaround a bug somewhere... */ - char *stripped_cwd = strchr(cwd, '/'); - if (stripped_cwd == NULL) stripped_cwd = cwd; PyObject *sysPath = PySys_GetObject("path"); - PyObject *path = PyUnicode_FromString(cwd); + PyObject *path = PyUnicode_FromString(stripped_cwd); PyList_Insert(sysPath, 0, path); FILE * mainpy = fopen(MAINPY, "r"); From 9357c295f300e343a66a333641b29baf919dbc43 Mon Sep 17 00:00:00 2001 From: DavidBuchanan314 Date: Mon, 28 May 2018 16:50:56 +0100 Subject: [PATCH 47/72] Add relative dir to import path --- source/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/main.c b/source/main.c index 0edbc1e..dd66faa 100644 --- a/source/main.c +++ b/source/main.c @@ -39,7 +39,7 @@ int main(int argc, char *argv[]) /* set up import path */ PyObject *sysPath = PySys_GetObject("path"); - PyObject *path = PyUnicode_FromString(stripped_cwd); + PyObject *path = PyUnicode_FromString(""); PyList_Insert(sysPath, 0, path); FILE * mainpy = fopen(MAINPY, "r"); From 51d7b41ddcb81dd3fd7ea64454fa0b399a24b773 Mon Sep 17 00:00:00 2001 From: AileenLumina Date: Mon, 28 May 2018 16:29:03 +0000 Subject: [PATCH 48/72] Bump version to v0.5.1-alpha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d7ef451..0ff91e0 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := $(TARGET) APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.5.0-alpha +APP_VERSION := 0.5.1-alpha #--------------------------------------------------------------------------------- # options for code generation From ca21d4a07bbcb1d4819af69d3d47f94ecb04a0d6 Mon Sep 17 00:00:00 2001 From: elnardu Date: Sun, 24 Jun 2018 19:29:32 +0600 Subject: [PATCH 49/72] Use gnu-sed on macos --- python_build/Makefile | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index efedfe0..1460d80 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -30,6 +30,13 @@ LIBS := -lnx # include and lib #--------------------------------------------------------------------------------- +# Use gnu-sed if it is macos +ifeq ($(UNAME_S),Darwin) + SED := gsed +else + SED := sed +endif + PYCONFIG_DIR := ../python_config LIBDIRS := $(PORTLIBS) $(LIBNX) @@ -42,7 +49,7 @@ ifndef PYVERS PYVERS := 3.5.3 endif -ANAME := libpython$(shell echo $(PYVERS) | sed 's/\([0-9]*\.\([0-9]*\)\).*/\1/').a +ANAME := libpython$(shell echo $(PYVERS) | $(SED) 's/\([0-9]*\.\([0-9]*\)\).*/\1/').a OUTDIR := nxpy$(PYVERS) @@ -54,6 +61,8 @@ _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 + + .PHONY: all clean all: distfPY @@ -67,7 +76,7 @@ distfPY: linkPY cp $(PYDIR)/Include/* $(OUTDIR)/include/nxpy cp $(PYDIR)/pyconfig.h $(OUTDIR)/include/nxpy/ cp $(PYDIR)/Lib/socket.py $(PYDIR)/Lib/socket.pyX - cat $(PYDIR)/Lib/socket.pyX | sed 's/'"'"'getpeername'"'"', //g' >$(PYDIR)/Lib/socket.py + cat $(PYDIR)/Lib/socket.pyX | $(SED) 's/'"'"'getpeername'"'"', //g' >$(PYDIR)/Lib/socket.py rm $(PYDIR)/Lib/socket.pyX cd $(PYDIR)/Lib && ls *.py -1 | xargs zip ../../$(OUTDIR)/python.zip && find json/ encodings/ html/ http/ test/ urllib/ collections/ email/ sqlite3/ logging/ xml/ importlib/ asyncio/ -type f | xargs zip ../../$(OUTDIR)/python.zip @@ -91,15 +100,15 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/getaddrinfo.c $(PYDIR)/Modules/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old - cat $(PYDIR)/Modules/posixmodule.c_old | sed 's/return utime(path, time)/errno=ENOENT; return -1/g' | sed 's/define LSTAT lstat/define LSTAT stat/' | sed '1s/^/#include /' | sed 's/access(path->narrow, mode)/1/' | sed 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | sed 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | sed 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | sed 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | sed 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c + cat $(PYDIR)/Modules/posixmodule.c_old | $(SED) 's/return utime(path, time)/errno=ENOENT; return -1/g' | $(SED) 's/define LSTAT lstat/define LSTAT stat/' | $(SED) '1s/^/#include /' | $(SED) 's/access(path->narrow, mode)/1/' | $(SED) 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | $(SED) 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | $(SED) 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | $(SED) 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | $(SED) 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old - cat $(PYDIR)/Modules/socketmodule.c_old | sed 's/send(s->sock_fd, buf, len, flags);/send(s->sock_fd, buf, len<4096?len:4096, flags);/g' | sed 's/ sizeof(addr->sa_data)/ 28/g' >$(PYDIR)/Modules/socketmodule.c + cat $(PYDIR)/Modules/socketmodule.c_old | $(SED) 's/send(s->sock_fd, buf, len, flags);/send(s->sock_fd, buf, len<4096?len:4096, flags);/g' | $(SED) 's/ sizeof(addr->sa_data)/ 28/g' >$(PYDIR)/Modules/socketmodule.c cp $(PYDIR)/Objects/exceptions.c $(PYDIR)/Objects/exceptions.c_old - cat $(PYDIR)/Objects/exceptions.c_old | sed 's/ESHUTDOWN/EPIPE/g' >$(PYDIR)/Objects/exceptions.c + cat $(PYDIR)/Objects/exceptions.c_old | $(SED) 's/ESHUTDOWN/EPIPE/g' >$(PYDIR)/Objects/exceptions.c cp $(PYDIR)/Python/pytime.c $(PYDIR)/Python/pytime.c_old - cat $(PYDIR)/Python/pytime.c_old | sed 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c + cat $(PYDIR)/Python/pytime.c_old | $(SED) 's/CLOCK_MONOTONIC/(clockid_t)4/g' >$(PYDIR)/Python/pytime.c cp $(PYDIR)/Makefile $(PYDIR)/Makefile_old - cat $(PYDIR)/Makefile_old | sed 's/-Werror=declaration-after-statement//' | sed 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile + cat $(PYDIR)/Makefile_old | $(SED) 's/-Werror=declaration-after-statement//' | $(SED) 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile touch soospatchPY compilePY: extractedPY patchPY @@ -107,12 +116,12 @@ compilePY: extractedPY patchPY patchPY: cloneNX cp $(PYDIR)/configure $(PYDIR)/configure_old - cat $(PYDIR)/configure_old | sed 's/ \*\-\*\-linux\*)/ \*\-\*\-linux\*\|aarch64\-none\-elf)/g' >$(PYDIR)/configure + cat $(PYDIR)/configure_old | $(SED) 's/ \*\-\*\-linux\*)/ \*\-\*\-linux\*\|aarch64\-none\-elf)/g' >$(PYDIR)/configure echo ac_cv_file__dev_ptmx=no >$(PYDIR)/config.site echo ac_cv_file__dev_ptc=no >>$(PYDIR)/config.site echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old - cat $(PYDIR)/Modules/Setup.dist_old | sed -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c' | sed 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | sed 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | sed "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" >$(PYDIR)/Modules/Setup.dist + cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" >$(PYDIR)/Modules/Setup.dist cp -r _nx-*/_nx $(PYDIR)/Modules/ touch patchPY From 37946763ba6084b15474219b091e2de636b352fb Mon Sep 17 00:00:00 2001 From: Elnard Utiushev Date: Sun, 24 Jun 2018 19:31:43 +0600 Subject: [PATCH 50/72] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index bf4c5fc..e5ef201 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ You don't have to compile PyNX, you can [just grab a release build](https://gith Compile PyNX using `make`. This will create a `build` directory and build everything in there. Compiling might take a while, grab a coffee or whatever in the meantime if you like. Afterwards, create a distributable version using `make dist`. It will appear in the `build` directory. +### Mac OS +You will have to install gnu-sed first(`brew install gnu-sed`). + ## Documentation Documentation can be found on [ReadTheDocs](https://nx-python.readthedocs.io/en/latest/). From 486e05cb5f2e0360e9690122eabcbcf89633827d Mon Sep 17 00:00:00 2001 From: Elnard Utiushev Date: Fri, 6 Jul 2018 16:17:17 +0600 Subject: [PATCH 51/72] Add thread support (#5) --- python_build/Makefile | 11 +- python_config/ceval_gil.h | 278 ++++++ python_config/condvar.h | 457 ++++++++++ python_config/pyconfig.h | 3 +- python_config/pylifecycle.c | 1604 +++++++++++++++++++++++++++++++++++ python_config/thread.c | 451 ++++++++++ python_config/thread_nx.h | 290 +++++++ 7 files changed, 3090 insertions(+), 4 deletions(-) create mode 100644 python_config/ceval_gil.h create mode 100644 python_config/condvar.h create mode 100644 python_config/pylifecycle.c create mode 100644 python_config/thread.c create mode 100644 python_config/thread_nx.h diff --git a/python_build/Makefile b/python_build/Makefile index 1460d80..0d6aeda 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -15,7 +15,7 @@ include $(DEVKITPRO)/libnx/switch_rules #--------------------------------------------------------------------------------- ARCH := -march=armv8-a -mtune=cortex-a57 -mtp=soft -fPIE -CFLAGS := -g -Wall -O2 -ffunction-sections \ +CFLAGS := -g -Wall -O2 -ffunction-sections -DDISABLE_TRACEMALLOC \ $(ARCH) $(DEFINES) CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions -std=gnu++11 @@ -98,6 +98,11 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/random.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/fileutils.c $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/thread.c $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/thread_nx.h $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/condvar.h $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/pylifecycle.c $(PYDIR)/Python/ + cp $(PYCONFIG_DIR)/ceval_gil.h $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/getaddrinfo.c $(PYDIR)/Modules/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | $(SED) 's/return utime(path, time)/errno=ENOENT; return -1/g' | $(SED) 's/define LSTAT lstat/define LSTAT stat/' | $(SED) '1s/^/#include /' | $(SED) 's/access(path->narrow, mode)/1/' | $(SED) 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | $(SED) 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | $(SED) 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | $(SED) 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | $(SED) 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c @@ -112,7 +117,7 @@ soospatchPY: compilePY touch soospatchPY compilePY: extractedPY patchPY - cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --without-threads --without-signal-module --disable-ipv6 --host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY + cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --with-threads --without-signal-module --disable-ipv6 -host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY patchPY: cloneNX cp $(PYDIR)/configure $(PYDIR)/configure_old @@ -121,7 +126,7 @@ patchPY: cloneNX echo ac_cv_file__dev_ptc=no >>$(PYDIR)/config.site echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old - cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" >$(PYDIR)/Modules/Setup.dist + cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c hashtable.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" | $(SED) "s/_tracemalloc/# _tracemalloc/" >$(PYDIR)/Modules/Setup.dist cp -r _nx-*/_nx $(PYDIR)/Modules/ touch patchPY diff --git a/python_config/ceval_gil.h b/python_config/ceval_gil.h new file mode 100644 index 0000000..0330167 --- /dev/null +++ b/python_config/ceval_gil.h @@ -0,0 +1,278 @@ +/* + * Implementation of the Global Interpreter Lock (GIL). + */ + +#include +#include + + +/* First some general settings */ + +/* microseconds (the Python API uses seconds, though) */ +#define DEFAULT_INTERVAL 5000 +static unsigned long gil_interval = DEFAULT_INTERVAL; +#define INTERVAL (gil_interval >= 1 ? gil_interval : 1) + +/* Enable if you want to force the switching of threads at least every `gil_interval` */ +#undef FORCE_SWITCHING +#define FORCE_SWITCHING + + +/* + Notes about the implementation: + + - The GIL is just a boolean variable (gil_locked) whose access is protected + by a mutex (gil_mutex), and whose changes are signalled by a condition + variable (gil_cond). gil_mutex is taken for short periods of time, + and therefore mostly uncontended. + + - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be + able to release the GIL on demand by another thread. A volatile boolean + variable (gil_drop_request) is used for that purpose, which is checked + at every turn of the eval loop. That variable is set after a wait of + `interval` microseconds on `gil_cond` has timed out. + + [Actually, another volatile boolean variable (eval_breaker) is used + which ORs several conditions into one. Volatile booleans are + sufficient as inter-thread signalling means since Python is run + on cache-coherent architectures only.] + + - A thread wanting to take the GIL will first let pass a given amount of + time (`interval` microseconds) before setting gil_drop_request. This + encourages a defined switching period, but doesn't enforce it since + opcodes can take an arbitrary time to execute. + + The `interval` value is available for the user to read and modify + using the Python API `sys.{get,set}switchinterval()`. + + - When a thread releases the GIL and gil_drop_request is set, that thread + ensures that another GIL-awaiting thread gets scheduled. + It does so by waiting on a condition variable (switch_cond) until + the value of gil_last_holder is changed to something else than its + own thread state pointer, indicating that another thread was able to + take the GIL. + + This is meant to prohibit the latency-adverse behaviour on multi-core + machines where one thread would speculatively release the GIL, but still + run and end up being the first to re-acquire it, making the "timeslices" + much longer than expected. + (Note: this mechanism is enabled with FORCE_SWITCHING above) +*/ + +#include "condvar.h" +#ifndef Py_HAVE_CONDVAR +#error You need either a POSIX-compatible or a Windows system! +#endif + +#define MUTEX_T PyMUTEX_T +#define MUTEX_INIT(mut) \ + if (PyMUTEX_INIT(&(mut))) { \ + Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); }; +#define MUTEX_FINI(mut) \ + if (PyMUTEX_FINI(&(mut))) { \ + Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); }; +#define MUTEX_LOCK(mut) \ + if (PyMUTEX_LOCK(&(mut))) { \ + Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); }; +#define MUTEX_UNLOCK(mut) \ + if (PyMUTEX_UNLOCK(&(mut))) { \ + Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); }; + +#define COND_T PyCOND_T + +#ifdef SWITCH +#define COND_INIT(cond, mut) \ + if (PyCOND_INIT(&(cond), &(mut))) { \ + Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; +#else +#define COND_INIT(cond) \ + if (PyCOND_INIT(&(cond))) { \ + Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; +#endif + +#define COND_FINI(cond) \ + if (PyCOND_FINI(&(cond))) { \ + Py_FatalError("PyCOND_FINI(" #cond ") failed"); }; +#define COND_SIGNAL(cond) \ + if (PyCOND_SIGNAL(&(cond))) { \ + Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); }; +#define COND_WAIT(cond, mut) \ + if (PyCOND_WAIT(&(cond), &(mut))) { \ + Py_FatalError("PyCOND_WAIT(" #cond ") failed"); }; +#define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \ + { \ + int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \ + if (r < 0) \ + Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \ + if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \ + timeout_result = 1; \ + else \ + timeout_result = 0; \ + } \ + + + +/* Whether the GIL is already taken (-1 if uninitialized). This is atomic + because it can be read without any lock taken in ceval.c. */ +static _Py_atomic_int gil_locked = {-1}; +/* Number of GIL switches since the beginning. */ +static unsigned long gil_switch_number = 0; +/* Last PyThreadState holding / having held the GIL. This helps us know + whether anyone else was scheduled after we dropped the GIL. */ +static _Py_atomic_address gil_last_holder = {0}; + +/* This condition variable allows one or several threads to wait until + the GIL is released. In addition, the mutex also protects the above + variables. */ +static COND_T gil_cond; +static MUTEX_T gil_mutex; + +#ifdef FORCE_SWITCHING +/* This condition variable helps the GIL-releasing thread wait for + a GIL-awaiting thread to be scheduled and take the GIL. */ +static COND_T switch_cond; +static MUTEX_T switch_mutex; +#endif + + +static int gil_created(void) +{ + return _Py_atomic_load_explicit(&gil_locked, _Py_memory_order_acquire) >= 0; +} + +static void create_gil(void) +{ + MUTEX_INIT(gil_mutex); +#ifdef FORCE_SWITCHING + MUTEX_INIT(switch_mutex); +#endif + COND_INIT(gil_cond, gil_mutex); +#ifdef FORCE_SWITCHING + COND_INIT(switch_cond, switch_mutex); +#endif + _Py_atomic_store_relaxed(&gil_last_holder, 0); + _Py_ANNOTATE_RWLOCK_CREATE(&gil_locked); + _Py_atomic_store_explicit(&gil_locked, 0, _Py_memory_order_release); +} + +static void destroy_gil(void) +{ + /* some pthread-like implementations tie the mutex to the cond + * and must have the cond destroyed first. + */ + COND_FINI(gil_cond); + MUTEX_FINI(gil_mutex); +#ifdef FORCE_SWITCHING + COND_FINI(switch_cond); + MUTEX_FINI(switch_mutex); +#endif + _Py_atomic_store_explicit(&gil_locked, -1, _Py_memory_order_release); + _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked); +} + +static void recreate_gil(void) +{ + _Py_ANNOTATE_RWLOCK_DESTROY(&gil_locked); + /* XXX should we destroy the old OS resources here? */ + create_gil(); +} + +static void drop_gil(PyThreadState *tstate) +{ + if (!_Py_atomic_load_relaxed(&gil_locked)) + Py_FatalError("drop_gil: GIL is not locked"); + /* tstate is allowed to be NULL (early interpreter init) */ + if (tstate != NULL) { + /* Sub-interpreter support: threads might have been switched + under our feet using PyThreadState_Swap(). Fix the GIL last + holder variable so that our heuristics work. */ + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); + } + + MUTEX_LOCK(gil_mutex); + _Py_ANNOTATE_RWLOCK_RELEASED(&gil_locked, /*is_write=*/1); + _Py_atomic_store_relaxed(&gil_locked, 0); + COND_SIGNAL(gil_cond); + MUTEX_UNLOCK(gil_mutex); + +#ifdef FORCE_SWITCHING + if (_Py_atomic_load_relaxed(&gil_drop_request) && tstate != NULL) { + MUTEX_LOCK(switch_mutex); + /* Not switched yet => wait */ + if ((PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder) == tstate) { + RESET_GIL_DROP_REQUEST(); + /* NOTE: if COND_WAIT does not atomically start waiting when + releasing the mutex, another thread can run through, take + the GIL and drop it again, and reset the condition + before we even had a chance to wait for it. */ + COND_WAIT(switch_cond, switch_mutex); + } + MUTEX_UNLOCK(switch_mutex); + } +#endif +} + +static void take_gil(PyThreadState *tstate) +{ + int err; + if (tstate == NULL) + Py_FatalError("take_gil: NULL tstate"); + + err = errno; + MUTEX_LOCK(gil_mutex); + + if (!_Py_atomic_load_relaxed(&gil_locked)) + goto _ready; + + while (_Py_atomic_load_relaxed(&gil_locked)) { + int timed_out = 0; + unsigned long saved_switchnum; + + saved_switchnum = gil_switch_number; + COND_TIMED_WAIT(gil_cond, gil_mutex, INTERVAL, timed_out); + /* If we timed out and no switch occurred in the meantime, it is time + to ask the GIL-holding thread to drop it. */ + if (timed_out && + _Py_atomic_load_relaxed(&gil_locked) && + gil_switch_number == saved_switchnum) { + SET_GIL_DROP_REQUEST(); + } + } +_ready: +#ifdef FORCE_SWITCHING + /* This mutex must be taken before modifying gil_last_holder (see drop_gil()). */ + MUTEX_LOCK(switch_mutex); +#endif + /* We now hold the GIL */ + _Py_atomic_store_relaxed(&gil_locked, 1); + _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil_locked, /*is_write=*/1); + + if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder)) { + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); + ++gil_switch_number; + } + +#ifdef FORCE_SWITCHING + COND_SIGNAL(switch_cond); + MUTEX_UNLOCK(switch_mutex); +#endif + if (_Py_atomic_load_relaxed(&gil_drop_request)) { + RESET_GIL_DROP_REQUEST(); + } + if (tstate->async_exc != NULL) { + _PyEval_SignalAsyncExc(); + } + + MUTEX_UNLOCK(gil_mutex); + errno = err; +} + +void _PyEval_SetSwitchInterval(unsigned long microseconds) +{ + gil_interval = microseconds; +} + +unsigned long _PyEval_GetSwitchInterval() +{ + return gil_interval; +} diff --git a/python_config/condvar.h b/python_config/condvar.h new file mode 100644 index 0000000..d3ea5f7 --- /dev/null +++ b/python_config/condvar.h @@ -0,0 +1,457 @@ +/* + * Portable condition variable support for windows and pthreads. + * Everything is inline, this header can be included where needed. + * + * APIs generally return 0 on success and non-zero on error, + * and the caller needs to use its platform's error mechanism to + * discover the error (errno, or GetLastError()) + * + * Note that some implementations cannot distinguish between a + * condition variable wait time-out and successful wait. Most often + * the difference is moot anyway since the wait condition must be + * re-checked. + * PyCOND_TIMEDWAIT, in addition to returning negative on error, + * thus returns 0 on regular success, 1 on timeout + * or 2 if it can't tell. + * + * There are at least two caveats with using these condition variables, + * due to the fact that they may be emulated with Semaphores on + * Windows: + * 1) While PyCOND_SIGNAL() will wake up at least one thread, we + * cannot currently guarantee that it will be one of the threads + * already waiting in a PyCOND_WAIT() call. It _could_ cause + * the wakeup of a subsequent thread to try a PyCOND_WAIT(), + * including the thread doing the PyCOND_SIGNAL() itself. + * The same applies to PyCOND_BROADCAST(), if N threads are waiting + * then at least N threads will be woken up, but not necessarily + * those already waiting. + * For this reason, don't make the scheduling assumption that a + * specific other thread will get the wakeup signal + * 2) The _mutex_ must be held when calling PyCOND_SIGNAL() and + * PyCOND_BROADCAST(). + * While e.g. the posix standard strongly recommends that the mutex + * associated with the condition variable is held when a + * pthread_cond_signal() call is made, this is not a hard requirement, + * although scheduling will not be "reliable" if it isn't. Here + * the mutex is used for internal synchronization of the emulated + * Condition Variable. + */ + +#ifndef _CONDVAR_H_ +#define _CONDVAR_H_ + +#include "Python.h" + +#ifndef _POSIX_THREADS +/* This means pthreads are not implemented in libc headers, hence the macro + not present in unistd.h. But they still can be implemented as an external + library (e.g. gnu pth in pthread emulation) */ +# ifdef HAVE_PTHREAD_H +# include /* _POSIX_THREADS */ +# endif +#endif + +#ifdef _POSIX_THREADS +/* + * POSIX support + */ +#define Py_HAVE_CONDVAR + +#include + +#define PyCOND_ADD_MICROSECONDS(tv, interval) \ +do { /* TODO: add overflow and truncation checks */ \ + tv.tv_usec += (long) interval; \ + tv.tv_sec += tv.tv_usec / 1000000; \ + tv.tv_usec %= 1000000; \ +} while (0) + +/* We assume all modern POSIX systems have gettimeofday() */ +#ifdef GETTIMEOFDAY_NO_TZ +#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv) +#else +#define PyCOND_GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL) +#endif + +/* The following functions return 0 on success, nonzero on error */ +#define PyMUTEX_T pthread_mutex_t +#define PyMUTEX_INIT(mut) pthread_mutex_init((mut), NULL) +#define PyMUTEX_FINI(mut) pthread_mutex_destroy(mut) +#define PyMUTEX_LOCK(mut) pthread_mutex_lock(mut) +#define PyMUTEX_UNLOCK(mut) pthread_mutex_unlock(mut) + +#define PyCOND_T pthread_cond_t +#define PyCOND_INIT(cond) pthread_cond_init((cond), NULL) +#define PyCOND_FINI(cond) pthread_cond_destroy(cond) +#define PyCOND_SIGNAL(cond) pthread_cond_signal(cond) +#define PyCOND_BROADCAST(cond) pthread_cond_broadcast(cond) +#define PyCOND_WAIT(cond, mut) pthread_cond_wait((cond), (mut)) + +/* return 0 for success, 1 on timeout, -1 on error */ +Py_LOCAL_INLINE(int) +PyCOND_TIMEDWAIT(PyCOND_T *cond, PyMUTEX_T *mut, PY_LONG_LONG us) +{ + int r; + struct timespec ts; + struct timeval deadline; + + PyCOND_GETTIMEOFDAY(&deadline); + PyCOND_ADD_MICROSECONDS(deadline, us); + ts.tv_sec = deadline.tv_sec; + ts.tv_nsec = deadline.tv_usec * 1000; + + r = pthread_cond_timedwait((cond), (mut), &ts); + if (r == ETIMEDOUT) + return 1; + else if (r) + return -1; + else + return 0; +} + +#elif defined(SWITCH) +#define Py_HAVE_CONDVAR +#include "switch/kernel/mutex.h" +#include "switch/kernel/svc.h" +#include "switch/kernel/condvar.h" +#include "switch/types.h" +#include "switch/arm/tls.h" +#include "switch/kernel/thread.h" + +/* The following functions return 0 on success, nonzero on error */ +#define PyMUTEX_T Mutex + +Py_LOCAL_INLINE(int) +PyMUTEX_INIT(PyMUTEX_T *mut) { + mutexInit(mut); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_FINI(PyMUTEX_T *mut) { + mutexInit(mut); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_LOCK(PyMUTEX_T *mut) { + mutexLock(mut); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_UNLOCK(PyMUTEX_T *mut) { + mutexUnlock(mut); + return 0; +} + + +#define PyCOND_T CondVar + +Py_LOCAL_INLINE(int) +PyCOND_INIT(PyCOND_T *cond, PyMUTEX_T *mut) { + condvarInit(cond, mut); + return 0; +} + +#define PyCOND_FINI(cond) 0 +#define PyCOND_SIGNAL(cond) condvarWakeOne((cond)) +#define PyCOND_BROADCAST(cond) condvarWakeAll((cond)) +#define PyCOND_WAIT(cond, mut) condvarWait((cond)) + +/* return 0 for success, 1 on timeout, -1 on error */ +Py_LOCAL_INLINE(int) +PyCOND_TIMEDWAIT(PyCOND_T *cond, PyMUTEX_T *mut, PY_LONG_LONG us) +{ + int r; + u64 ns = us * 1000; // microseconds to nanoseconds + + r = condvarWaitTimeout((cond), ns); + if (r == 0xEA01) + return 1; + else if (r) + return -1; + else + return 0; +} + + +#elif defined(NT_THREADS) +/* + * Windows (XP, 2003 server and later, as well as (hopefully) CE) support + * + * Emulated condition variables ones that work with XP and later, plus + * example native support on VISTA and onwards. + */ +#define Py_HAVE_CONDVAR + + +/* include windows if it hasn't been done before */ +#define WIN32_LEAN_AND_MEAN +#include + +/* options */ +/* non-emulated condition variables are provided for those that want + * to target Windows Vista. Modify this macro to enable them. + */ +#ifndef _PY_EMULATED_WIN_CV +#define _PY_EMULATED_WIN_CV 1 /* use emulated condition variables */ +#endif + +/* fall back to emulation if not targeting Vista */ +#if !defined NTDDI_VISTA || NTDDI_VERSION < NTDDI_VISTA +#undef _PY_EMULATED_WIN_CV +#define _PY_EMULATED_WIN_CV 1 +#endif + + +#if _PY_EMULATED_WIN_CV + +/* The mutex is a CriticalSection object and + The condition variables is emulated with the help of a semaphore. + Semaphores are available on Windows XP (2003 server) and later. + We use a Semaphore rather than an auto-reset event, because although + an auto-resent event might appear to solve the lost-wakeup bug (race + condition between releasing the outer lock and waiting) because it + maintains state even though a wait hasn't happened, there is still + a lost wakeup problem if more than one thread are interrupted in the + critical place. A semaphore solves that, because its state is counted, + not Boolean. + Because it is ok to signal a condition variable with no one + waiting, we need to keep track of the number of + waiting threads. Otherwise, the semaphore's state could rise + without bound. This also helps reduce the number of "spurious wakeups" + that would otherwise happen. + + This implementation still has the problem that the threads woken + with a "signal" aren't necessarily those that are already + waiting. It corresponds to listing 2 in: + http://birrell.org/andrew/papers/ImplementingCVs.pdf + + Generic emulations of the pthread_cond_* API using + earlier Win32 functions can be found on the Web. + The following read can be give background information to these issues, + but the implementations are all broken in some way. + http://www.cse.wustl.edu/~schmidt/win32-cv-1.html +*/ + +typedef CRITICAL_SECTION PyMUTEX_T; + +Py_LOCAL_INLINE(int) +PyMUTEX_INIT(PyMUTEX_T *cs) +{ + InitializeCriticalSection(cs); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_FINI(PyMUTEX_T *cs) +{ + DeleteCriticalSection(cs); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_LOCK(PyMUTEX_T *cs) +{ + EnterCriticalSection(cs); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_UNLOCK(PyMUTEX_T *cs) +{ + LeaveCriticalSection(cs); + return 0; +} + +/* The ConditionVariable object. From XP onwards it is easily emulated with + * a Semaphore + */ + +typedef struct _PyCOND_T +{ + HANDLE sem; + int waiting; /* to allow PyCOND_SIGNAL to be a no-op */ +} PyCOND_T; + +Py_LOCAL_INLINE(int) +PyCOND_INIT(PyCOND_T *cv) +{ + /* A semaphore with a "large" max value, The positive value + * is only needed to catch those "lost wakeup" events and + * race conditions when a timed wait elapses. + */ + cv->sem = CreateSemaphore(NULL, 0, 100000, NULL); + if (cv->sem==NULL) + return -1; + cv->waiting = 0; + return 0; +} + +Py_LOCAL_INLINE(int) +PyCOND_FINI(PyCOND_T *cv) +{ + return CloseHandle(cv->sem) ? 0 : -1; +} + +/* this implementation can detect a timeout. Returns 1 on timeout, + * 0 otherwise (and -1 on error) + */ +Py_LOCAL_INLINE(int) +_PyCOND_WAIT_MS(PyCOND_T *cv, PyMUTEX_T *cs, DWORD ms) +{ + DWORD wait; + cv->waiting++; + PyMUTEX_UNLOCK(cs); + /* "lost wakeup bug" would occur if the caller were interrupted here, + * but we are safe because we are using a semaphore which has an internal + * count. + */ + wait = WaitForSingleObjectEx(cv->sem, ms, FALSE); + PyMUTEX_LOCK(cs); + if (wait != WAIT_OBJECT_0) + --cv->waiting; + /* Here we have a benign race condition with PyCOND_SIGNAL. + * When failure occurs or timeout, it is possible that + * PyCOND_SIGNAL also decrements this value + * and signals releases the mutex. This is benign because it + * just means an extra spurious wakeup for a waiting thread. + * ('waiting' corresponds to the semaphore's "negative" count and + * we may end up with e.g. (waiting == -1 && sem.count == 1). When + * a new thread comes along, it will pass right throuhgh, having + * adjusted it to (waiting == 0 && sem.count == 0). + */ + + if (wait == WAIT_FAILED) + return -1; + /* return 0 on success, 1 on timeout */ + return wait != WAIT_OBJECT_0; +} + +Py_LOCAL_INLINE(int) +PyCOND_WAIT(PyCOND_T *cv, PyMUTEX_T *cs) +{ + int result = _PyCOND_WAIT_MS(cv, cs, INFINITE); + return result >= 0 ? 0 : result; +} + +Py_LOCAL_INLINE(int) +PyCOND_TIMEDWAIT(PyCOND_T *cv, PyMUTEX_T *cs, PY_LONG_LONG us) +{ + return _PyCOND_WAIT_MS(cv, cs, (DWORD)(us/1000)); +} + +Py_LOCAL_INLINE(int) +PyCOND_SIGNAL(PyCOND_T *cv) +{ + /* this test allows PyCOND_SIGNAL to be a no-op unless required + * to wake someone up, thus preventing an unbounded increase of + * the semaphore's internal counter. + */ + if (cv->waiting > 0) { + /* notifying thread decreases the cv->waiting count so that + * a delay between notify and actual wakeup of the target thread + * doesn't cause a number of extra ReleaseSemaphore calls. + */ + cv->waiting--; + return ReleaseSemaphore(cv->sem, 1, NULL) ? 0 : -1; + } + return 0; +} + +Py_LOCAL_INLINE(int) +PyCOND_BROADCAST(PyCOND_T *cv) +{ + int waiting = cv->waiting; + if (waiting > 0) { + cv->waiting = 0; + return ReleaseSemaphore(cv->sem, waiting, NULL) ? 0 : -1; + } + return 0; +} + +#else + +/* Use native Win7 primitives if build target is Win7 or higher */ + +/* SRWLOCK is faster and better than CriticalSection */ +typedef SRWLOCK PyMUTEX_T; + +Py_LOCAL_INLINE(int) +PyMUTEX_INIT(PyMUTEX_T *cs) +{ + InitializeSRWLock(cs); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_FINI(PyMUTEX_T *cs) +{ + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_LOCK(PyMUTEX_T *cs) +{ + AcquireSRWLockExclusive(cs); + return 0; +} + +Py_LOCAL_INLINE(int) +PyMUTEX_UNLOCK(PyMUTEX_T *cs) +{ + ReleaseSRWLockExclusive(cs); + return 0; +} + + +typedef CONDITION_VARIABLE PyCOND_T; + +Py_LOCAL_INLINE(int) +PyCOND_INIT(PyCOND_T *cv) +{ + InitializeConditionVariable(cv); + return 0; +} +Py_LOCAL_INLINE(int) +PyCOND_FINI(PyCOND_T *cv) +{ + return 0; +} + +Py_LOCAL_INLINE(int) +PyCOND_WAIT(PyCOND_T *cv, PyMUTEX_T *cs) +{ + return SleepConditionVariableSRW(cv, cs, INFINITE, 0) ? 0 : -1; +} + +/* This implementation makes no distinction about timeouts. Signal + * 2 to indicate that we don't know. + */ +Py_LOCAL_INLINE(int) +PyCOND_TIMEDWAIT(PyCOND_T *cv, PyMUTEX_T *cs, PY_LONG_LONG us) +{ + return SleepConditionVariableSRW(cv, cs, (DWORD)(us/1000), 0) ? 2 : -1; +} + +Py_LOCAL_INLINE(int) +PyCOND_SIGNAL(PyCOND_T *cv) +{ + WakeConditionVariable(cv); + return 0; +} + +Py_LOCAL_INLINE(int) +PyCOND_BROADCAST(PyCOND_T *cv) +{ + WakeAllConditionVariable(cv); + return 0; +} + + +#endif /* _PY_EMULATED_WIN_CV */ + +#endif /* _POSIX_THREADS, NT_THREADS */ + +#endif /* _CONDVAR_H_ */ diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index 51af129..0f9c0c6 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -6,6 +6,7 @@ #define Py_PYCONFIG_H + /* Define if building universal (internal helper macro) */ /* #undef AC_APPLE_UNIVERSAL_BUILD */ @@ -1384,7 +1385,7 @@ #define WITH_PYMALLOC 1 /* Define if you want to compile in rudimentary thread support */ -/* #undef WITH_THREAD */ +#define WITH_THREAD /* Define to profile with the Pentium timestamp counter */ /* #undef WITH_TSC */ diff --git a/python_config/pylifecycle.c b/python_config/pylifecycle.c new file mode 100644 index 0000000..17d127c --- /dev/null +++ b/python_config/pylifecycle.c @@ -0,0 +1,1604 @@ +/* Python interpreter top-level routines, including init/exit */ + +#include "Python.h" + +#include "Python-ast.h" +#undef Yield /* undefine macro conflicting with winbase.h */ +#include "grammar.h" +#include "node.h" +#include "token.h" +#include "parsetok.h" +#include "errcode.h" +#include "code.h" +#include "symtable.h" +#include "ast.h" +#include "marshal.h" +#include "osdefs.h" +#include + +#ifdef HAVE_SIGNAL_H +#include +#endif + +#ifdef MS_WINDOWS +#include "malloc.h" /* for alloca */ +#endif + +#ifdef HAVE_LANGINFO_H +#include +#endif + +#ifdef MS_WINDOWS +#undef BYTE +#include "windows.h" +#endif + +_Py_IDENTIFIER(flush); +_Py_IDENTIFIER(name); +_Py_IDENTIFIER(stdin); +_Py_IDENTIFIER(stdout); +_Py_IDENTIFIER(stderr); + +#ifdef __cplusplus +extern "C" { +#endif + +extern wchar_t *Py_GetPath(void); + +extern grammar _PyParser_Grammar; /* From graminit.c */ + +/* Forward */ +static void initmain(PyInterpreterState *interp); +static int initfsencoding(PyInterpreterState *interp); +static void initsite(void); +static int initstdio(void); +static void initsigs(void); +static void call_py_exitfuncs(void); +static void wait_for_thread_shutdown(void); +static void call_ll_exitfuncs(void); +extern int _PyUnicode_Init(void); +extern int _PyStructSequence_Init(void); +extern void _PyUnicode_Fini(void); +extern int _PyLong_Init(void); +extern void PyLong_Fini(void); +extern int _PyFaulthandler_Init(void); +extern void _PyFaulthandler_Fini(void); +extern void _PyHash_Fini(void); + +#ifndef DISABLE_TRACEMALLOC +extern int _PyTraceMalloc_Init(void); +extern int _PyTraceMalloc_Fini(void); +#endif + +#ifdef WITH_THREAD +extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); +extern void _PyGILState_Fini(void); +#endif /* WITH_THREAD */ + +/* Global configuration variable declarations are in pydebug.h */ +/* XXX (ncoghlan): move those declarations to pylifecycle.h? */ +int Py_DebugFlag; /* Needed by parser.c */ +int Py_VerboseFlag; /* Needed by import.c */ +int Py_QuietFlag; /* Needed by sysmodule.c */ +int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */ +int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ +int Py_OptimizeFlag = 0; /* Needed by compile.c */ +int Py_NoSiteFlag; /* Suppress 'import site' */ +int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ +int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ +int Py_FrozenFlag; /* Needed by getpath.c */ +int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ +int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ +int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ +int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ +int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ +int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */ + +PyThreadState *_Py_Finalizing = NULL; + +/* Hack to force loading of object files */ +int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t) = \ + PyOS_mystrnicmp; /* Python/pystrcmp.o */ + +/* PyModule_GetWarningsModule is no longer necessary as of 2.6 +since _warnings is builtin. This API should not be used. */ +PyObject * +PyModule_GetWarningsModule(void) +{ + return PyImport_ImportModule("warnings"); +} + +static int initialized = 0; + +/* API to access the initialized flag -- useful for esoteric use */ + +int +Py_IsInitialized(void) +{ + return initialized; +} + +/* Helper to allow an embedding application to override the normal + * mechanism that attempts to figure out an appropriate IO encoding + */ + +static char *_Py_StandardStreamEncoding = NULL; +static char *_Py_StandardStreamErrors = NULL; + +int +Py_SetStandardStreamEncoding(const char *encoding, const char *errors) +{ + if (Py_IsInitialized()) { + /* This is too late to have any effect */ + return -1; + } + /* Can't call PyErr_NoMemory() on errors, as Python hasn't been + * initialised yet. + * + * However, the raw memory allocators are initialised appropriately + * as C static variables, so _PyMem_RawStrdup is OK even though + * Py_Initialize hasn't been called yet. + */ + if (encoding) { + _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding); + if (!_Py_StandardStreamEncoding) { + return -2; + } + } + if (errors) { + _Py_StandardStreamErrors = _PyMem_RawStrdup(errors); + if (!_Py_StandardStreamErrors) { + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + } + return -3; + } + } + return 0; +} + +/* Global initializations. Can be undone by Py_Finalize(). Don't + call this twice without an intervening Py_Finalize() call. When + initializations fail, a fatal error is issued and the function does + not return. On return, the first thread and interpreter state have + been created. + + Locking: you must hold the interpreter lock while calling this. + (If the lock has not yet been initialized, that's equivalent to + having the lock, but you cannot use multiple threads.) + +*/ + +static int +add_flag(int flag, const char *envs) +{ + int env = atoi(envs); + if (flag < env) + flag = env; + if (flag < 1) + flag = 1; + return flag; +} + +static char* +get_codec_name(const char *encoding) +{ + char *name_utf8, *name_str; + PyObject *codec, *name = NULL; + + codec = _PyCodec_Lookup(encoding); + if (!codec) + goto error; + + name = _PyObject_GetAttrId(codec, &PyId_name); + Py_CLEAR(codec); + if (!name) + goto error; + + name_utf8 = _PyUnicode_AsString(name); + if (name_utf8 == NULL) + goto error; + name_str = _PyMem_RawStrdup(name_utf8); + Py_DECREF(name); + if (name_str == NULL) { + PyErr_NoMemory(); + return NULL; + } + return name_str; + +error: + Py_XDECREF(codec); + Py_XDECREF(name); + return NULL; +} + +static char* +get_locale_encoding(void) +{ +#ifdef MS_WINDOWS + char codepage[100]; + PyOS_snprintf(codepage, sizeof(codepage), "cp%d", GetACP()); + return get_codec_name(codepage); +#elif defined(HAVE_LANGINFO_H) && defined(CODESET) + char* codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + PyErr_SetString(PyExc_ValueError, "CODESET is not set or empty"); + return NULL; + } + return get_codec_name(codeset); +#else + PyErr_SetNone(PyExc_NotImplementedError); + return NULL; +#endif +} + +static void +import_init(PyInterpreterState *interp, PyObject *sysmod) +{ + PyObject *importlib; + PyObject *impmod; + PyObject *sys_modules; + PyObject *value; + + /* Import _importlib through its frozen version, _frozen_importlib. */ + if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { + Py_FatalError("Py_Initialize: can't import _frozen_importlib"); + } + else if (Py_VerboseFlag) { + PySys_FormatStderr("import _frozen_importlib # frozen\n"); + } + importlib = PyImport_AddModule("_frozen_importlib"); + if (importlib == NULL) { + Py_FatalError("Py_Initialize: couldn't get _frozen_importlib from " + "sys.modules"); + } + interp->importlib = importlib; + Py_INCREF(interp->importlib); + + /* Import the _imp module */ + impmod = PyInit_imp(); + if (impmod == NULL) { + Py_FatalError("Py_Initialize: can't import _imp"); + } + else if (Py_VerboseFlag) { + PySys_FormatStderr("import _imp # builtin\n"); + } + sys_modules = PyImport_GetModuleDict(); + if (Py_VerboseFlag) { + PySys_FormatStderr("import sys # builtin\n"); + } + if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) { + Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); + } + + /* Install importlib as the implementation of import */ + value = PyObject_CallMethod(importlib, "_install", "OO", sysmod, impmod); + if (value == NULL) { + PyErr_Print(); + Py_FatalError("Py_Initialize: importlib install failed"); + } + Py_DECREF(value); + Py_DECREF(impmod); + + _PyImportZip_Init(); +} + + +void +_Py_InitializeEx_Private(int install_sigs, int install_importlib) +{ + PyInterpreterState *interp; + PyThreadState *tstate; + PyObject *bimod, *sysmod, *pstderr; + char *p; + extern void _Py_ReadyTypes(void); + + if (initialized) + return; + initialized = 1; + _Py_Finalizing = NULL; + +#if defined(HAVE_LANGINFO_H) && defined(HAVE_SETLOCALE) + /* Set up the LC_CTYPE locale, so we can obtain + the locale's charset without having to switch + locales. */ + setlocale(LC_CTYPE, ""); +#endif + + if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0') + Py_DebugFlag = add_flag(Py_DebugFlag, p); + if ((p = Py_GETENV("PYTHONVERBOSE")) && *p != '\0') + Py_VerboseFlag = add_flag(Py_VerboseFlag, p); + if ((p = Py_GETENV("PYTHONOPTIMIZE")) && *p != '\0') + Py_OptimizeFlag = add_flag(Py_OptimizeFlag, p); + if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0') + Py_DontWriteBytecodeFlag = add_flag(Py_DontWriteBytecodeFlag, p); + /* The variable is only tested for existence here; _PyRandom_Init will + check its value further. */ + if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0') + Py_HashRandomizationFlag = add_flag(Py_HashRandomizationFlag, p); + + _PyRandom_Init(); + + interp = PyInterpreterState_New(); + if (interp == NULL) + Py_FatalError("Py_Initialize: can't make first interpreter"); + + tstate = PyThreadState_New(interp); + if (tstate == NULL) + Py_FatalError("Py_Initialize: can't make first thread"); + (void) PyThreadState_Swap(tstate); + +#ifdef WITH_THREAD + /* We can't call _PyEval_FiniThreads() in Py_Finalize because + destroying the GIL might fail when it is being referenced from + another running thread (see issue #9901). + Instead we destroy the previously created GIL here, which ensures + that we can call Py_Initialize / Py_Finalize multiple times. */ + _PyEval_FiniThreads(); + + /* Auto-thread-state API */ + _PyGILState_Init(interp, tstate); +#endif /* WITH_THREAD */ + + _Py_ReadyTypes(); + + if (!_PyFrame_Init()) + Py_FatalError("Py_Initialize: can't init frames"); + + if (!_PyLong_Init()) + Py_FatalError("Py_Initialize: can't init longs"); + + if (!PyByteArray_Init()) + Py_FatalError("Py_Initialize: can't init bytearray"); + + if (!_PyFloat_Init()) + Py_FatalError("Py_Initialize: can't init float"); + + interp->modules = PyDict_New(); + if (interp->modules == NULL) + Py_FatalError("Py_Initialize: can't make modules dictionary"); + + /* Init Unicode implementation; relies on the codec registry */ + if (_PyUnicode_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize unicode"); + if (_PyStructSequence_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize structseq"); + + bimod = _PyBuiltin_Init(); + if (bimod == NULL) + Py_FatalError("Py_Initialize: can't initialize builtins modules"); + _PyImport_FixupBuiltin(bimod, "builtins"); + interp->builtins = PyModule_GetDict(bimod); + if (interp->builtins == NULL) + Py_FatalError("Py_Initialize: can't initialize builtins dict"); + Py_INCREF(interp->builtins); + + /* initialize builtin exceptions */ + _PyExc_Init(bimod); + + sysmod = _PySys_Init(); + if (sysmod == NULL) + Py_FatalError("Py_Initialize: can't initialize sys"); + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + Py_FatalError("Py_Initialize: can't initialize sys dict"); + Py_INCREF(interp->sysdict); + _PyImport_FixupBuiltin(sysmod, "sys"); + PySys_SetPath(Py_GetPath()); + PyDict_SetItemString(interp->sysdict, "modules", + interp->modules); + + /* Set up a preliminary stderr printer until we have enough + infrastructure for the io module in place. */ + pstderr = PyFile_NewStdPrinter(fileno(stderr)); + if (pstderr == NULL) + Py_FatalError("Py_Initialize: can't set preliminary stderr"); + _PySys_SetObjectId(&PyId_stderr, pstderr); + PySys_SetObject("__stderr__", pstderr); + Py_DECREF(pstderr); + + _PyImport_Init(); + + _PyImportHooks_Init(); + + /* Initialize _warnings. */ + _PyWarnings_Init(); + + if (!install_importlib) + return; + + if (_PyTime_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize time"); + + import_init(interp, sysmod); + + /* initialize the faulthandler module */ + if (_PyFaulthandler_Init()) + Py_FatalError("Py_Initialize: can't initialize faulthandler"); + + if (initfsencoding(interp) < 0) + Py_FatalError("Py_Initialize: unable to load the file system codec"); + + if (install_sigs) + initsigs(); /* Signal handling stuff, including initintr() */ + + #ifndef DISABLE_TRACEMALLOC + if (_PyTraceMalloc_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize tracemalloc"); + #endif + + initmain(interp); /* Module __main__ */ + if (initstdio() < 0) + Py_FatalError( + "Py_Initialize: can't initialize sys standard streams"); + + /* Initialize warnings. */ + if (PySys_HasWarnOptions()) { + PyObject *warnings_module = PyImport_ImportModule("warnings"); + if (warnings_module == NULL) { + fprintf(stderr, "'import warnings' failed; traceback:\n"); + PyErr_Print(); + } + Py_XDECREF(warnings_module); + } + + if (!Py_NoSiteFlag) + initsite(); /* Module site */ +} + +void +Py_InitializeEx(int install_sigs) +{ + _Py_InitializeEx_Private(install_sigs, 1); +} + +void +Py_Initialize(void) +{ + Py_InitializeEx(1); +} + + +#ifdef COUNT_ALLOCS +extern void dump_counts(FILE*); +#endif + +/* Flush stdout and stderr */ + +static int +file_is_closed(PyObject *fobj) +{ + int r; + PyObject *tmp = PyObject_GetAttrString(fobj, "closed"); + if (tmp == NULL) { + PyErr_Clear(); + return 0; + } + r = PyObject_IsTrue(tmp); + Py_DECREF(tmp); + if (r < 0) + PyErr_Clear(); + return r > 0; +} + +static void +flush_std_files(void) +{ + PyObject *fout = _PySys_GetObjectId(&PyId_stdout); + PyObject *ferr = _PySys_GetObjectId(&PyId_stderr); + PyObject *tmp; + + if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { + tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); + if (tmp == NULL) + PyErr_WriteUnraisable(fout); + else + Py_DECREF(tmp); + } + + if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { + tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); + if (tmp == NULL) + PyErr_Clear(); + else + Py_DECREF(tmp); + } +} + +/* Undo the effect of Py_Initialize(). + + Beware: if multiple interpreter and/or thread states exist, these + are not wiped out; only the current thread and interpreter state + are deleted. But since everything else is deleted, those other + interpreter and thread states should no longer be used. + + (XXX We should do better, e.g. wipe out all interpreters and + threads.) + + Locking: as above. + +*/ + +void +Py_Finalize(void) +{ + PyInterpreterState *interp; + PyThreadState *tstate; + + if (!initialized) + return; + + wait_for_thread_shutdown(); + + /* The interpreter is still entirely intact at this point, and the + * exit funcs may be relying on that. In particular, if some thread + * or exit func is still waiting to do an import, the import machinery + * expects Py_IsInitialized() to return true. So don't say the + * interpreter is uninitialized until after the exit funcs have run. + * Note that Threading.py uses an exit func to do a join on all the + * threads created thru it, so this also protects pending imports in + * the threads created via Threading. + */ + call_py_exitfuncs(); + + /* Get current thread state and interpreter pointer */ + tstate = PyThreadState_GET(); + interp = tstate->interp; + + /* Remaining threads (e.g. daemon threads) will automatically exit + after taking the GIL (in PyEval_RestoreThread()). */ + _Py_Finalizing = tstate; + initialized = 0; + + /* Flush sys.stdout and sys.stderr */ + flush_std_files(); + + /* Disable signal handling */ + PyOS_FiniInterrupts(); + + /* Collect garbage. This may call finalizers; it's nice to call these + * before all modules are destroyed. + * XXX If a __del__ or weakref callback is triggered here, and tries to + * XXX import a module, bad things can happen, because Python no + * XXX longer believes it's initialized. + * XXX Fatal Python error: Interpreter not initialized (version mismatch?) + * XXX is easy to provoke that way. I've also seen, e.g., + * XXX Exception exceptions.ImportError: 'No module named sha' + * XXX in ignored + * XXX but I'm unclear on exactly how that one happens. In any case, + * XXX I haven't seen a real-life report of either of these. + */ + PyGC_Collect(); +#ifdef COUNT_ALLOCS + /* With COUNT_ALLOCS, it helps to run GC multiple times: + each collection might release some types from the type + list, so they become garbage. */ + while (PyGC_Collect() > 0) + /* nothing */; +#endif + /* Destroy all modules */ + PyImport_Cleanup(); + + /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ + flush_std_files(); + + /* Collect final garbage. This disposes of cycles created by + * class definitions, for example. + * XXX This is disabled because it caused too many problems. If + * XXX a __del__ or weakref callback triggers here, Python code has + * XXX a hard time running, because even the sys module has been + * XXX cleared out (sys.stdout is gone, sys.excepthook is gone, etc). + * XXX One symptom is a sequence of information-free messages + * XXX coming from threads (if a __del__ or callback is invoked, + * XXX other threads can execute too, and any exception they encounter + * XXX triggers a comedy of errors as subsystem after subsystem + * XXX fails to find what it *expects* to find in sys to help report + * XXX the exception and consequent unexpected failures). I've also + * XXX seen segfaults then, after adding print statements to the + * XXX Python code getting called. + */ +#if 0 + PyGC_Collect(); +#endif + + #ifndef DISABLE_TRACEMALLOC + /* Disable tracemalloc after all Python objects have been destroyed, + so it is possible to use tracemalloc in objects destructor. */ + _PyTraceMalloc_Fini(); + #endif + + /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ + _PyImport_Fini(); + + /* Cleanup typeobject.c's internal caches. */ + _PyType_Fini(); + + /* unload faulthandler module */ + _PyFaulthandler_Fini(); + + /* Debugging stuff */ +#ifdef COUNT_ALLOCS + dump_counts(stdout); +#endif + /* dump hash stats */ + _PyHash_Fini(); + + _PY_DEBUG_PRINT_TOTAL_REFS(); + +#ifdef Py_TRACE_REFS + /* Display all objects still alive -- this can invoke arbitrary + * __repr__ overrides, so requires a mostly-intact interpreter. + * Alas, a lot of stuff may still be alive now that will be cleaned + * up later. + */ + if (Py_GETENV("PYTHONDUMPREFS")) + _Py_PrintReferences(stderr); +#endif /* Py_TRACE_REFS */ + + /* Clear interpreter state and all thread states. */ + PyInterpreterState_Clear(interp); + + /* Now we decref the exception classes. After this point nothing + can raise an exception. That's okay, because each Fini() method + below has been checked to make sure no exceptions are ever + raised. + */ + + _PyExc_Fini(); + + /* Sundry finalizers */ + PyMethod_Fini(); + PyFrame_Fini(); + PyCFunction_Fini(); + PyTuple_Fini(); + PyList_Fini(); + PySet_Fini(); + PyBytes_Fini(); + PyByteArray_Fini(); + PyLong_Fini(); + PyFloat_Fini(); + PyDict_Fini(); + PySlice_Fini(); + _PyGC_Fini(); + _PyRandom_Fini(); + + /* Cleanup Unicode implementation */ + _PyUnicode_Fini(); + + /* reset file system default encoding */ + if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) { + PyMem_RawFree((char*)Py_FileSystemDefaultEncoding); + Py_FileSystemDefaultEncoding = NULL; + } + + /* XXX Still allocated: + - various static ad-hoc pointers to interned strings + - int and float free list blocks + - whatever various modules and libraries allocate + */ + + PyGrammar_RemoveAccelerators(&_PyParser_Grammar); + + /* Cleanup auto-thread-state */ +#ifdef WITH_THREAD + _PyGILState_Fini(); +#endif /* WITH_THREAD */ + + /* Delete current thread. After this, many C API calls become crashy. */ + PyThreadState_Swap(NULL); + PyInterpreterState_Delete(interp); + +#ifdef Py_TRACE_REFS + /* Display addresses (& refcnts) of all objects still alive. + * An address can be used to find the repr of the object, printed + * above by _Py_PrintReferences. + */ + if (Py_GETENV("PYTHONDUMPREFS")) + _Py_PrintReferenceAddresses(stderr); +#endif /* Py_TRACE_REFS */ +#ifdef PYMALLOC_DEBUG + if (Py_GETENV("PYTHONMALLOCSTATS")) + _PyObject_DebugMallocStats(stderr); +#endif + + call_ll_exitfuncs(); +} + +/* Create and initialize a new interpreter and thread, and return the + new thread. This requires that Py_Initialize() has been called + first. + + Unsuccessful initialization yields a NULL pointer. Note that *no* + exception information is available even in this case -- the + exception information is held in the thread, and there is no + thread. + + Locking: as above. + +*/ + +PyThreadState * +Py_NewInterpreter(void) +{ + PyInterpreterState *interp; + PyThreadState *tstate, *save_tstate; + PyObject *bimod, *sysmod; + + if (!initialized) + Py_FatalError("Py_NewInterpreter: call Py_Initialize first"); + + interp = PyInterpreterState_New(); + if (interp == NULL) + return NULL; + + tstate = PyThreadState_New(interp); + if (tstate == NULL) { + PyInterpreterState_Delete(interp); + return NULL; + } + + save_tstate = PyThreadState_Swap(tstate); + + /* XXX The following is lax in error checking */ + + interp->modules = PyDict_New(); + + bimod = _PyImport_FindBuiltin("builtins"); + if (bimod != NULL) { + interp->builtins = PyModule_GetDict(bimod); + if (interp->builtins == NULL) + goto handle_error; + Py_INCREF(interp->builtins); + } + + /* initialize builtin exceptions */ + _PyExc_Init(bimod); + + sysmod = _PyImport_FindBuiltin("sys"); + if (bimod != NULL && sysmod != NULL) { + PyObject *pstderr; + + interp->sysdict = PyModule_GetDict(sysmod); + if (interp->sysdict == NULL) + goto handle_error; + Py_INCREF(interp->sysdict); + PySys_SetPath(Py_GetPath()); + PyDict_SetItemString(interp->sysdict, "modules", + interp->modules); + /* Set up a preliminary stderr printer until we have enough + infrastructure for the io module in place. */ + pstderr = PyFile_NewStdPrinter(fileno(stderr)); + if (pstderr == NULL) + Py_FatalError("Py_Initialize: can't set preliminary stderr"); + _PySys_SetObjectId(&PyId_stderr, pstderr); + PySys_SetObject("__stderr__", pstderr); + Py_DECREF(pstderr); + + _PyImportHooks_Init(); + + import_init(interp, sysmod); + + if (initfsencoding(interp) < 0) + goto handle_error; + + if (initstdio() < 0) + Py_FatalError( + "Py_Initialize: can't initialize sys standard streams"); + initmain(interp); + if (!Py_NoSiteFlag) + initsite(); + } + + if (!PyErr_Occurred()) + return tstate; + +handle_error: + /* Oops, it didn't work. Undo it all. */ + + PyErr_PrintEx(0); + PyThreadState_Clear(tstate); + PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); + PyInterpreterState_Delete(interp); + + return NULL; +} + +/* Delete an interpreter and its last thread. This requires that the + given thread state is current, that the thread has no remaining + frames, and that it is its interpreter's only remaining thread. + It is a fatal error to violate these constraints. + + (Py_Finalize() doesn't have these constraints -- it zaps + everything, regardless.) + + Locking: as above. + +*/ + +void +Py_EndInterpreter(PyThreadState *tstate) +{ + PyInterpreterState *interp = tstate->interp; + + if (tstate != PyThreadState_GET()) + Py_FatalError("Py_EndInterpreter: thread is not current"); + if (tstate->frame != NULL) + Py_FatalError("Py_EndInterpreter: thread still has a frame"); + + wait_for_thread_shutdown(); + + if (tstate != interp->tstate_head || tstate->next != NULL) + Py_FatalError("Py_EndInterpreter: not the last thread"); + + PyImport_Cleanup(); + PyInterpreterState_Clear(interp); + PyThreadState_Swap(NULL); + PyInterpreterState_Delete(interp); +} + +#ifdef MS_WINDOWS +static wchar_t *progname = L"python"; +#else +static wchar_t *progname = L"python3"; +#endif + +void +Py_SetProgramName(wchar_t *pn) +{ + if (pn && *pn) + progname = pn; +} + +wchar_t * +Py_GetProgramName(void) +{ + return progname; +} + +static wchar_t *default_home = NULL; +static wchar_t env_home[MAXPATHLEN+1]; + +void +Py_SetPythonHome(wchar_t *home) +{ + default_home = home; +} + +wchar_t * +Py_GetPythonHome(void) +{ + wchar_t *home = default_home; + if (home == NULL && !Py_IgnoreEnvironmentFlag) { + char* chome = Py_GETENV("PYTHONHOME"); + if (chome) { + size_t size = Py_ARRAY_LENGTH(env_home); + size_t r = mbstowcs(env_home, chome, size); + if (r != (size_t)-1 && r < size) + home = env_home; + } + + } + return home; +} + +/* Create __main__ module */ + +static void +initmain(PyInterpreterState *interp) +{ + PyObject *m, *d, *loader; + m = PyImport_AddModule("__main__"); + if (m == NULL) + Py_FatalError("can't create __main__ module"); + d = PyModule_GetDict(m); + if (PyDict_GetItemString(d, "__builtins__") == NULL) { + PyObject *bimod = PyImport_ImportModule("builtins"); + if (bimod == NULL) { + Py_FatalError("Failed to retrieve builtins module"); + } + if (PyDict_SetItemString(d, "__builtins__", bimod) < 0) { + Py_FatalError("Failed to initialize __main__.__builtins__"); + } + Py_DECREF(bimod); + } + /* Main is a little special - imp.is_builtin("__main__") will return + * False, but BuiltinImporter is still the most appropriate initial + * setting for its __loader__ attribute. A more suitable value will + * be set if __main__ gets further initialized later in the startup + * process. + */ + loader = PyDict_GetItemString(d, "__loader__"); + if (loader == NULL || loader == Py_None) { + PyObject *loader = PyObject_GetAttrString(interp->importlib, + "BuiltinImporter"); + if (loader == NULL) { + Py_FatalError("Failed to retrieve BuiltinImporter"); + } + if (PyDict_SetItemString(d, "__loader__", loader) < 0) { + Py_FatalError("Failed to initialize __main__.__loader__"); + } + Py_DECREF(loader); + } +} + +static int +initfsencoding(PyInterpreterState *interp) +{ + PyObject *codec; + + if (Py_FileSystemDefaultEncoding == NULL) + { + Py_FileSystemDefaultEncoding = get_locale_encoding(); + if (Py_FileSystemDefaultEncoding == NULL) + Py_FatalError("Py_Initialize: Unable to get the locale encoding"); + + Py_HasFileSystemDefaultEncoding = 0; + interp->fscodec_initialized = 1; + return 0; + } + + /* the encoding is mbcs, utf-8 or ascii */ + codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding); + if (!codec) { + /* Such error can only occurs in critical situations: no more + * memory, import a module of the standard library failed, + * etc. */ + return -1; + } + Py_DECREF(codec); + interp->fscodec_initialized = 1; + return 0; +} + +/* Import the site module (not into __main__ though) */ + +static void +initsite(void) +{ + PyObject *m; + m = PyImport_ImportModule("site"); + if (m == NULL) { + fprintf(stderr, "Failed to import the site module\n"); + PyErr_Print(); + Py_Finalize(); + exit(1); + } + else { + Py_DECREF(m); + } +} + +/* Check if a file descriptor is valid or not. + Return 0 if the file descriptor is invalid, return non-zero otherwise. */ +static int +is_valid_fd(int fd) +{ + int fd2; + if (fd < 0 || !_PyVerify_fd(fd)) + return 0; + _Py_BEGIN_SUPPRESS_IPH + fd2 = dup(fd); + if (fd2 >= 0) + close(fd2); + _Py_END_SUPPRESS_IPH + return fd2 >= 0; +} + +/* returns Py_None if the fd is not valid */ +static PyObject* +create_stdio(PyObject* io, + int fd, int write_mode, char* name, + char* encoding, char* errors) +{ + PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; + const char* mode; + const char* newline; + PyObject *line_buffering; + int buffering, isatty; + _Py_IDENTIFIER(open); + _Py_IDENTIFIER(isatty); + _Py_IDENTIFIER(TextIOWrapper); + _Py_IDENTIFIER(mode); + + if (!is_valid_fd(fd)) + Py_RETURN_NONE; + + /* stdin is always opened in buffered mode, first because it shouldn't + make a difference in common use cases, second because TextIOWrapper + depends on the presence of a read1() method which only exists on + buffered streams. + */ + if (Py_UnbufferedStdioFlag && write_mode) + buffering = 0; + else + buffering = -1; + if (write_mode) + mode = "wb"; + else + mode = "rb"; + buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi", + fd, mode, buffering, + Py_None, Py_None, Py_None, 0); + if (buf == NULL) + goto error; + + if (buffering) { + _Py_IDENTIFIER(raw); + raw = _PyObject_GetAttrId(buf, &PyId_raw); + if (raw == NULL) + goto error; + } + else { + raw = buf; + Py_INCREF(raw); + } + + text = PyUnicode_FromString(name); + if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0) + goto error; + res = _PyObject_CallMethodId(raw, &PyId_isatty, ""); + if (res == NULL) + goto error; + isatty = PyObject_IsTrue(res); + Py_DECREF(res); + if (isatty == -1) + goto error; + if (isatty || Py_UnbufferedStdioFlag) + line_buffering = Py_True; + else + line_buffering = Py_False; + + Py_CLEAR(raw); + Py_CLEAR(text); + +#ifdef MS_WINDOWS + /* sys.stdin: enable universal newline mode, translate "\r\n" and "\r" + newlines to "\n". + sys.stdout and sys.stderr: translate "\n" to "\r\n". */ + newline = NULL; +#else + /* sys.stdin: split lines at "\n". + sys.stdout and sys.stderr: don't translate newlines (use "\n"). */ + newline = "\n"; +#endif + + stream = _PyObject_CallMethodId(io, &PyId_TextIOWrapper, "OsssO", + buf, encoding, errors, + newline, line_buffering); + Py_CLEAR(buf); + if (stream == NULL) + goto error; + + if (write_mode) + mode = "w"; + else + mode = "r"; + text = PyUnicode_FromString(mode); + if (!text || _PyObject_SetAttrId(stream, &PyId_mode, text) < 0) + goto error; + Py_CLEAR(text); + return stream; + +error: + Py_XDECREF(buf); + Py_XDECREF(stream); + Py_XDECREF(text); + Py_XDECREF(raw); + + if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) { + /* Issue #24891: the file descriptor was closed after the first + is_valid_fd() check was called. Ignore the OSError and set the + stream to None. */ + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; +} + +/* Initialize sys.stdin, stdout, stderr and builtins.open */ +static int +initstdio(void) +{ + PyObject *iomod = NULL, *wrapper; + PyObject *bimod = NULL; + PyObject *m; + PyObject *std = NULL; + int status = 0, fd; + PyObject * encoding_attr; + char *pythonioencoding = NULL, *encoding, *errors; + + /* Hack to avoid a nasty recursion issue when Python is invoked + in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ + if ((m = PyImport_ImportModule("encodings.utf_8")) == NULL) { + goto error; + } + Py_DECREF(m); + + if (!(m = PyImport_ImportModule("encodings.latin_1"))) { + goto error; + } + Py_DECREF(m); + + if (!(bimod = PyImport_ImportModule("builtins"))) { + goto error; + } + + if (!(iomod = PyImport_ImportModule("io"))) { + goto error; + } + if (!(wrapper = PyObject_GetAttrString(iomod, "OpenWrapper"))) { + goto error; + } + + /* Set builtins.open */ + if (PyObject_SetAttrString(bimod, "open", wrapper) == -1) { + Py_DECREF(wrapper); + goto error; + } + Py_DECREF(wrapper); + + encoding = _Py_StandardStreamEncoding; + errors = _Py_StandardStreamErrors; + if (!encoding || !errors) { + pythonioencoding = Py_GETENV("PYTHONIOENCODING"); + if (pythonioencoding) { + char *err; + pythonioencoding = _PyMem_Strdup(pythonioencoding); + if (pythonioencoding == NULL) { + PyErr_NoMemory(); + goto error; + } + err = strchr(pythonioencoding, ':'); + if (err) { + *err = '\0'; + err++; + if (*err && !errors) { + errors = err; + } + } + if (*pythonioencoding && !encoding) { + encoding = pythonioencoding; + } + } + if (!errors && !(pythonioencoding && *pythonioencoding)) { + /* When the LC_CTYPE locale is the POSIX locale ("C locale"), + stdin and stdout use the surrogateescape error handler by + default, instead of the strict error handler. */ + char *loc = setlocale(LC_CTYPE, NULL); + if (loc != NULL && strcmp(loc, "C") == 0) + errors = "surrogateescape"; + } + } + + /* Set sys.stdin */ + fd = fileno(stdin); + /* Under some conditions stdin, stdout and stderr may not be connected + * and fileno() may point to an invalid file descriptor. For example + * GUI apps don't have valid standard streams by default. + */ + std = create_stdio(iomod, fd, 0, "", encoding, errors); + if (std == NULL) + goto error; + PySys_SetObject("__stdin__", std); + _PySys_SetObjectId(&PyId_stdin, std); + Py_DECREF(std); + + /* Set sys.stdout */ + fd = fileno(stdout); + std = create_stdio(iomod, fd, 1, "", encoding, errors); + if (std == NULL) + goto error; + PySys_SetObject("__stdout__", std); + _PySys_SetObjectId(&PyId_stdout, std); + Py_DECREF(std); + +#if 1 /* Disable this if you have trouble debugging bootstrap stuff */ + /* Set sys.stderr, replaces the preliminary stderr */ + fd = fileno(stderr); + std = create_stdio(iomod, fd, 1, "", encoding, "backslashreplace"); + if (std == NULL) + goto error; + + /* Same as hack above, pre-import stderr's codec to avoid recursion + when import.c tries to write to stderr in verbose mode. */ + encoding_attr = PyObject_GetAttrString(std, "encoding"); + if (encoding_attr != NULL) { + const char * std_encoding; + std_encoding = _PyUnicode_AsString(encoding_attr); + if (std_encoding != NULL) { + PyObject *codec_info = _PyCodec_Lookup(std_encoding); + Py_XDECREF(codec_info); + } + Py_DECREF(encoding_attr); + } + PyErr_Clear(); /* Not a fatal error if codec isn't available */ + + if (PySys_SetObject("__stderr__", std) < 0) { + Py_DECREF(std); + goto error; + } + if (_PySys_SetObjectId(&PyId_stderr, std) < 0) { + Py_DECREF(std); + goto error; + } + Py_DECREF(std); +#endif + + if (0) { + error: + status = -1; + } + + /* We won't need them anymore. */ + if (_Py_StandardStreamEncoding) { + PyMem_RawFree(_Py_StandardStreamEncoding); + _Py_StandardStreamEncoding = NULL; + } + if (_Py_StandardStreamErrors) { + PyMem_RawFree(_Py_StandardStreamErrors); + _Py_StandardStreamErrors = NULL; + } + PyMem_Free(pythonioencoding); + Py_XDECREF(bimod); + Py_XDECREF(iomod); + return status; +} + + +static void +_Py_FatalError_DumpTracebacks(int fd) +{ + PyThreadState *tstate; + +#ifdef WITH_THREAD + /* PyGILState_GetThisThreadState() works even if the GIL was released */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_GET(); +#endif + if (tstate == NULL) { + /* _Py_DumpTracebackThreads() requires the thread state to display + * frames */ + return; + } + + fputc('\n', stderr); + fflush(stderr); + + /* display the current Python stack */ + _Py_DumpTracebackThreads(fd, tstate->interp, tstate); +} + +/* Print the current exception (if an exception is set) with its traceback, + or display the current Python stack. + + Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is + called on catastrophic cases. + + Return 1 if the traceback was displayed, 0 otherwise. */ + +static int +_Py_FatalError_PrintExc(int fd) +{ + PyObject *ferr, *res; + PyObject *exception, *v, *tb; + int has_tb; + + if (PyThreadState_GET() == NULL) { + /* The GIL is released: trying to acquire it is likely to deadlock, + just give up. */ + return 0; + } + + PyErr_Fetch(&exception, &v, &tb); + if (exception == NULL) { + /* No current exception */ + return 0; + } + + ferr = _PySys_GetObjectId(&PyId_stderr); + if (ferr == NULL || ferr == Py_None) { + /* sys.stderr is not set yet or set to None, + no need to try to display the exception */ + return 0; + } + + PyErr_NormalizeException(&exception, &v, &tb); + if (tb == NULL) { + tb = Py_None; + Py_INCREF(tb); + } + PyException_SetTraceback(v, tb); + if (exception == NULL) { + /* PyErr_NormalizeException() failed */ + return 0; + } + + has_tb = (tb != Py_None); + PyErr_Display(exception, v, tb); + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + /* sys.stderr may be buffered: call sys.stderr.flush() */ + res = _PyObject_CallMethodId(ferr, &PyId_flush, ""); + if (res == NULL) + PyErr_Clear(); + else + Py_DECREF(res); + + return has_tb; +} + +/* Print fatal error message and abort */ + +void +Py_FatalError(const char *msg) +{ + const int fd = fileno(stderr); + static int reentrant = 0; +#ifdef MS_WINDOWS + size_t len; + WCHAR* buffer; + size_t i; +#endif + + if (reentrant) { + /* Py_FatalError() caused a second fatal error. + Example: flush_std_files() raises a recursion error. */ + goto exit; + } + reentrant = 1; + + fprintf(stderr, "Fatal Python error: %s\n", msg); + fflush(stderr); /* it helps in Windows debug build */ + + /* Print the exception (if an exception is set) with its traceback, + * or display the current Python stack. */ + if (!_Py_FatalError_PrintExc(fd)) + _Py_FatalError_DumpTracebacks(fd); + + /* The main purpose of faulthandler is to display the traceback. We already + * did our best to display it. So faulthandler can now be disabled. + * (Don't trigger it on abort().) */ + _PyFaulthandler_Fini(); + + /* Check if the current Python thread hold the GIL */ + if (PyThreadState_GET() != NULL) { + /* Flush sys.stdout and sys.stderr */ + flush_std_files(); + } + +#ifdef MS_WINDOWS + len = strlen(msg); + + /* Convert the message to wchar_t. This uses a simple one-to-one + conversion, assuming that the this error message actually uses ASCII + only. If this ceases to be true, we will have to convert. */ + buffer = alloca( (len+1) * (sizeof *buffer)); + for( i=0; i<=len; ++i) + buffer[i] = msg[i]; + OutputDebugStringW(L"Fatal Python error: "); + OutputDebugStringW(buffer); + OutputDebugStringW(L"\n"); +#endif /* MS_WINDOWS */ + +exit: +#if defined(MS_WINDOWS) && defined(_DEBUG) + DebugBreak(); +#endif + abort(); +} + +/* Clean up and exit */ + +#ifdef WITH_THREAD +#include "pythread.h" +#endif + +static void (*pyexitfunc)(void) = NULL; +/* For the atexit module. */ +void _Py_PyAtExit(void (*func)(void)) +{ + pyexitfunc = func; +} + +static void +call_py_exitfuncs(void) +{ + if (pyexitfunc == NULL) + return; + + (*pyexitfunc)(); + PyErr_Clear(); +} + +/* Wait until threading._shutdown completes, provided + the threading module was imported in the first place. + The shutdown routine will wait until all non-daemon + "threading" threads have completed. */ +static void +wait_for_thread_shutdown(void) +{ +#ifdef WITH_THREAD + _Py_IDENTIFIER(_shutdown); + PyObject *result; + PyThreadState *tstate = PyThreadState_GET(); + PyObject *threading = PyMapping_GetItemString(tstate->interp->modules, + "threading"); + if (threading == NULL) { + /* threading not imported */ + PyErr_Clear(); + return; + } + result = _PyObject_CallMethodId(threading, &PyId__shutdown, ""); + if (result == NULL) { + PyErr_WriteUnraisable(threading); + } + else { + Py_DECREF(result); + } + Py_DECREF(threading); +#endif +} + +#define NEXITFUNCS 32 +static void (*exitfuncs[NEXITFUNCS])(void); +static int nexitfuncs = 0; + +int Py_AtExit(void (*func)(void)) +{ + if (nexitfuncs >= NEXITFUNCS) + return -1; + exitfuncs[nexitfuncs++] = func; + return 0; +} + +static void +call_ll_exitfuncs(void) +{ + while (nexitfuncs > 0) + (*exitfuncs[--nexitfuncs])(); + + fflush(stdout); + fflush(stderr); +} + +void +Py_Exit(int sts) +{ + Py_Finalize(); + + exit(sts); +} + +static void +initsigs(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_IGN); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_IGN); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_IGN); +#endif + PyOS_InitInterrupts(); /* May imply initsignal() */ + if (PyErr_Occurred()) { + Py_FatalError("Py_Initialize: can't import signal"); + } +} + + +/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. + * + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ +void +_Py_RestoreSignals(void) +{ +#ifdef SIGPIPE + PyOS_setsig(SIGPIPE, SIG_DFL); +#endif +#ifdef SIGXFZ + PyOS_setsig(SIGXFZ, SIG_DFL); +#endif +#ifdef SIGXFSZ + PyOS_setsig(SIGXFSZ, SIG_DFL); +#endif +} + + +/* + * The file descriptor fd is considered ``interactive'' if either + * a) isatty(fd) is TRUE, or + * b) the -i flag was given, and the filename associated with + * the descriptor is NULL or "" or "???". + */ +int +Py_FdIsInteractive(FILE *fp, const char *filename) +{ + if (isatty((int)fileno(fp))) + return 1; + if (!Py_InteractiveFlag) + return 0; + return (filename == NULL) || + (strcmp(filename, "") == 0) || + (strcmp(filename, "???") == 0); +} + + +/* Wrappers around sigaction() or signal(). */ + +PyOS_sighandler_t +PyOS_getsig(int sig) +{ +#ifdef HAVE_SIGACTION + struct sigaction context; + if (sigaction(sig, NULL, &context) == -1) + return SIG_ERR; + return context.sa_handler; +#else + PyOS_sighandler_t handler; +/* Special signal handling for the secure CRT in Visual Studio 2005 */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 + switch (sig) { + /* Only these signals are valid */ + case SIGINT: + case SIGILL: + case SIGFPE: + case SIGSEGV: + case SIGTERM: + case SIGBREAK: + case SIGABRT: + break; + /* Don't call signal() with other values or it will assert */ + default: + return SIG_ERR; + } +#endif /* _MSC_VER && _MSC_VER >= 1400 */ + handler = signal(sig, SIG_IGN); + if (handler != SIG_ERR) + signal(sig, handler); + return handler; +#endif +} + +/* + * All of the code in this function must only use async-signal-safe functions, + * listed at `man 7 signal` or + * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. + */ +PyOS_sighandler_t +PyOS_setsig(int sig, PyOS_sighandler_t handler) +{ +#ifdef HAVE_SIGACTION + /* Some code in Modules/signalmodule.c depends on sigaction() being + * used here if HAVE_SIGACTION is defined. Fix that if this code + * changes to invalidate that assumption. + */ + struct sigaction context, ocontext; + context.sa_handler = handler; + sigemptyset(&context.sa_mask); + context.sa_flags = 0; + if (sigaction(sig, &context, &ocontext) == -1) + return SIG_ERR; + return ocontext.sa_handler; +#else + PyOS_sighandler_t oldhandler; + oldhandler = signal(sig, handler); +#ifdef HAVE_SIGINTERRUPT + siginterrupt(sig, 1); +#endif + return oldhandler; +#endif +} + +#ifdef __cplusplus +} +#endif diff --git a/python_config/thread.c b/python_config/thread.c new file mode 100644 index 0000000..b653037 --- /dev/null +++ b/python_config/thread.c @@ -0,0 +1,451 @@ + +/* Thread package. + This is intended to be usable independently from Python. + The implementation for system foobar is in a file thread_foobar.h + which is included by this file dependent on config settings. + Stuff shared by all thread_*.h files is collected here. */ + +#include "Python.h" + +#ifndef _POSIX_THREADS +/* This means pthreads are not implemented in libc headers, hence the macro + not present in unistd.h. But they still can be implemented as an external + library (e.g. gnu pth in pthread emulation) */ +# ifdef HAVE_PTHREAD_H +# include /* _POSIX_THREADS */ +# endif +#endif + +#ifndef DONT_HAVE_STDIO_H +#include +#endif + +#include + +#include "pythread.h" + +#ifndef _POSIX_THREADS + +/* Check if we're running on HP-UX and _SC_THREADS is defined. If so, then + enough of the Posix threads package is implemented to support python + threads. + + This is valid for HP-UX 11.23 running on an ia64 system. If needed, add + a check of __ia64 to verify that we're running on an ia64 system instead + of a pa-risc system. +*/ +#ifdef __hpux +#ifdef _SC_THREADS +#define _POSIX_THREADS +#endif +#endif + +#endif /* _POSIX_THREADS */ + + +#ifdef Py_DEBUG +static int thread_debug = 0; +#define dprintf(args) (void)((thread_debug & 1) && printf args) +#define d2printf(args) ((thread_debug & 8) && printf args) +#else +#define dprintf(args) +#define d2printf(args) +#endif + +static int initialized; + +static void PyThread__init_thread(void); /* Forward */ + +void +PyThread_init_thread(void) +{ +#ifdef Py_DEBUG + char *p = Py_GETENV("PYTHONTHREADDEBUG"); + + if (p) { + if (*p) + thread_debug = atoi(p); + else + thread_debug = 1; + } +#endif /* Py_DEBUG */ + if (initialized) + return; + initialized = 1; + dprintf(("PyThread_init_thread called\n")); + PyThread__init_thread(); +} + +/* Support for runtime thread stack size tuning. + A value of 0 means using the platform's default stack size + or the size specified by the THREAD_STACK_SIZE macro. */ +static size_t _pythread_stacksize = 0; + +#ifdef _POSIX_THREADS +#define PYTHREAD_NAME "pthread" +#include "thread_pthread.h" +#endif + +#ifdef NT_THREADS +#define PYTHREAD_NAME "nt" +#include "thread_nt.h" +#endif + +#ifdef SWITCH +#define PYTHREAD_NAME "nx" +#include "thread_nx.h" +#endif + +/* +#ifdef FOOBAR_THREADS +#include "thread_foobar.h" +#endif +*/ + +/* return the current thread stack size */ +size_t +PyThread_get_stacksize(void) +{ + return _pythread_stacksize; +} + +/* Only platforms defining a THREAD_SET_STACKSIZE() macro + in thread_.h support changing the stack size. + Return 0 if stack size is valid, + -1 if stack size value is invalid, + -2 if setting stack size is not supported. */ +int +PyThread_set_stacksize(size_t size) +{ +#if defined(THREAD_SET_STACKSIZE) + return THREAD_SET_STACKSIZE(size); +#else + return -2; +#endif +} + +#ifndef Py_HAVE_NATIVE_TLS +/* If the platform has not supplied a platform specific + TLS implementation, provide our own. + + This code stolen from "thread_sgi.h", where it was the only + implementation of an existing Python TLS API. +*/ +/* ------------------------------------------------------------------------ +Per-thread data ("key") support. + +Use PyThread_create_key() to create a new key. This is typically shared +across threads. + +Use PyThread_set_key_value(thekey, value) to associate void* value with +thekey in the current thread. Each thread has a distinct mapping of thekey +to a void* value. Caution: if the current thread already has a mapping +for thekey, value is ignored. + +Use PyThread_get_key_value(thekey) to retrieve the void* value associated +with thekey in the current thread. This returns NULL if no value is +associated with thekey in the current thread. + +Use PyThread_delete_key_value(thekey) to forget the current thread's associated +value for thekey. PyThread_delete_key(thekey) forgets the values associated +with thekey across *all* threads. + +While some of these functions have error-return values, none set any +Python exception. + +None of the functions does memory management on behalf of the void* values. +You need to allocate and deallocate them yourself. If the void* values +happen to be PyObject*, these functions don't do refcount operations on +them either. + +The GIL does not need to be held when calling these functions; they supply +their own locking. This isn't true of PyThread_create_key(), though (see +next paragraph). + +There's a hidden assumption that PyThread_create_key() will be called before +any of the other functions are called. There's also a hidden assumption +that calls to PyThread_create_key() are serialized externally. +------------------------------------------------------------------------ */ + +/* A singly-linked list of struct key objects remembers all the key->value + * associations. File static keyhead heads the list. keymutex is used + * to enforce exclusion internally. + */ +struct key { + /* Next record in the list, or NULL if this is the last record. */ + struct key *next; + + /* The thread id, according to PyThread_get_thread_ident(). */ + long id; + + /* The key and its associated value. */ + int key; + void *value; +}; + +static struct key *keyhead = NULL; +static PyThread_type_lock keymutex = NULL; +static int nkeys = 0; /* PyThread_create_key() hands out nkeys+1 next */ + +/* Internal helper. + * If the current thread has a mapping for key, the appropriate struct key* + * is returned. NB: value is ignored in this case! + * If there is no mapping for key in the current thread, then: + * If value is NULL, NULL is returned. + * Else a mapping of key to value is created for the current thread, + * and a pointer to a new struct key* is returned; except that if + * malloc() can't find room for a new struct key*, NULL is returned. + * So when value==NULL, this acts like a pure lookup routine, and when + * value!=NULL, this acts like dict.setdefault(), returning an existing + * mapping if one exists, else creating a new mapping. + * + * Caution: this used to be too clever, trying to hold keymutex only + * around the "p->next = keyhead; keyhead = p" pair. That allowed + * another thread to mutate the list, via key deletion, concurrent with + * find_key() crawling over the list. Hilarity ensued. For example, when + * the for-loop here does "p = p->next", p could end up pointing at a + * record that PyThread_delete_key_value() was concurrently free()'ing. + * That could lead to anything, from failing to find a key that exists, to + * segfaults. Now we lock the whole routine. + */ +static struct key * +find_key(int set_value, int key, void *value) +{ + struct key *p, *prev_p; + long id = PyThread_get_thread_ident(); + + if (!keymutex) + return NULL; + PyThread_acquire_lock(keymutex, 1); + prev_p = NULL; + for (p = keyhead; p != NULL; p = p->next) { + if (p->id == id && p->key == key) { + if (set_value) + p->value = value; + goto Done; + } + /* Sanity check. These states should never happen but if + * they do we must abort. Otherwise we'll end up spinning + * in a tight loop with the lock held. A similar check is done + * in pystate.c tstate_delete_common(). */ + if (p == prev_p) + Py_FatalError("tls find_key: small circular list(!)"); + prev_p = p; + if (p->next == keyhead) + Py_FatalError("tls find_key: circular list(!)"); + } + if (!set_value && value == NULL) { + assert(p == NULL); + goto Done; + } + p = (struct key *)PyMem_RawMalloc(sizeof(struct key)); + if (p != NULL) { + p->id = id; + p->key = key; + p->value = value; + p->next = keyhead; + keyhead = p; + } + Done: + PyThread_release_lock(keymutex); + return p; +} + +/* Return a new key. This must be called before any other functions in + * this family, and callers must arrange to serialize calls to this + * function. No violations are detected. + */ +int +PyThread_create_key(void) +{ + /* All parts of this function are wrong if it's called by multiple + * threads simultaneously. + */ + if (keymutex == NULL) + keymutex = PyThread_allocate_lock(); + return ++nkeys; +} + +/* Forget the associations for key across *all* threads. */ +void +PyThread_delete_key(int key) +{ + struct key *p, **q; + + PyThread_acquire_lock(keymutex, 1); + q = &keyhead; + while ((p = *q) != NULL) { + if (p->key == key) { + *q = p->next; + PyMem_RawFree((void *)p); + /* NB This does *not* free p->value! */ + } + else + q = &p->next; + } + PyThread_release_lock(keymutex); +} + +int +PyThread_set_key_value(int key, void *value) +{ + struct key *p; + + p = find_key(1, key, value); + if (p == NULL) + return -1; + else + return 0; +} + +/* Retrieve the value associated with key in the current thread, or NULL + * if the current thread doesn't have an association for key. + */ +void * +PyThread_get_key_value(int key) +{ + struct key *p = find_key(0, key, NULL); + + if (p == NULL) + return NULL; + else + return p->value; +} + +/* Forget the current thread's association for key, if any. */ +void +PyThread_delete_key_value(int key) +{ + long id = PyThread_get_thread_ident(); + struct key *p, **q; + + PyThread_acquire_lock(keymutex, 1); + q = &keyhead; + while ((p = *q) != NULL) { + if (p->key == key && p->id == id) { + *q = p->next; + PyMem_RawFree((void *)p); + /* NB This does *not* free p->value! */ + break; + } + else + q = &p->next; + } + PyThread_release_lock(keymutex); +} + +/* Forget everything not associated with the current thread id. + * This function is called from PyOS_AfterFork(). It is necessary + * because other thread ids which were in use at the time of the fork + * may be reused for new threads created in the forked process. + */ +void +PyThread_ReInitTLS(void) +{ + long id = PyThread_get_thread_ident(); + struct key *p, **q; + + if (!keymutex) + return; + + /* As with interpreter_lock in PyEval_ReInitThreads() + we just create a new lock without freeing the old one */ + keymutex = PyThread_allocate_lock(); + + /* Delete all keys which do not match the current thread id */ + q = &keyhead; + while ((p = *q) != NULL) { + if (p->id != id) { + *q = p->next; + PyMem_RawFree((void *)p); + /* NB This does *not* free p->value! */ + } + else + q = &p->next; + } +} + +#endif /* Py_HAVE_NATIVE_TLS */ + +PyDoc_STRVAR(threadinfo__doc__, +"sys.thread_info\n\ +\n\ +A struct sequence holding information about the thread implementation."); + +static PyStructSequence_Field threadinfo_fields[] = { + {"name", "name of the thread implementation"}, + {"lock", "name of the lock implementation"}, + {"version", "name and version of the thread library"}, + {0} +}; + +static PyStructSequence_Desc threadinfo_desc = { + "sys.thread_info", /* name */ + threadinfo__doc__, /* doc */ + threadinfo_fields, /* fields */ + 3 +}; + +static PyTypeObject ThreadInfoType; + +PyObject* +PyThread_GetInfo(void) +{ + PyObject *threadinfo, *value; + int pos = 0; +#if (defined(_POSIX_THREADS) && defined(HAVE_CONFSTR) \ + && defined(_CS_GNU_LIBPTHREAD_VERSION)) + char buffer[255]; + int len; +#endif + + if (ThreadInfoType.tp_name == 0) { + if (PyStructSequence_InitType2(&ThreadInfoType, &threadinfo_desc) < 0) + return NULL; + } + + threadinfo = PyStructSequence_New(&ThreadInfoType); + if (threadinfo == NULL) + return NULL; + + value = PyUnicode_FromString(PYTHREAD_NAME); + if (value == NULL) { + Py_DECREF(threadinfo); + return NULL; + } + PyStructSequence_SET_ITEM(threadinfo, pos++, value); + +#ifdef _POSIX_THREADS +#ifdef USE_SEMAPHORES + value = PyUnicode_FromString("semaphore"); +#else + value = PyUnicode_FromString("mutex+cond"); +#endif + if (value == NULL) { + Py_DECREF(threadinfo); + return NULL; + } +#else + Py_INCREF(Py_None); + value = Py_None; +#endif + PyStructSequence_SET_ITEM(threadinfo, pos++, value); + +#if (defined(_POSIX_THREADS) && defined(HAVE_CONFSTR) \ + && defined(_CS_GNU_LIBPTHREAD_VERSION)) + value = NULL; + len = confstr(_CS_GNU_LIBPTHREAD_VERSION, buffer, sizeof(buffer)); + if (1 < len && (size_t)len < sizeof(buffer)) { + value = PyUnicode_DecodeFSDefaultAndSize(buffer, len-1); + if (value == NULL) + PyErr_Clear(); + } + if (value == NULL) +#endif + { + Py_INCREF(Py_None); + value = Py_None; + } + PyStructSequence_SET_ITEM(threadinfo, pos++, value); + return threadinfo; +} diff --git a/python_config/thread_nx.h b/python_config/thread_nx.h new file mode 100644 index 0000000..04ff8b7 --- /dev/null +++ b/python_config/thread_nx.h @@ -0,0 +1,290 @@ +#include "switch/kernel/mutex.h" +#include "switch/kernel/svc.h" +#include "switch/kernel/condvar.h" +#include "switch/types.h" +#include "switch/arm/tls.h" +#include "switch/kernel/thread.h" + +#define THREAD_STACK_SIZE 0x4000 +#define THREAD_PRIORITY 0x20 +#define THREAD_CPU_ID -2 + + +// copied from nx/source/internal.h +#define THREADVARS_MAGIC 0x21545624 // !TV$ + +// This structure is exactly 0x20 bytes, if more is needed modify getThreadVars() below +typedef struct { + // Magic value used to check if the struct is initialized + u32 magic; + + // Thread handle, for mutexes + Handle handle; + + // Pointer to the current thread (if exists) + Thread* thread_ptr; + + // Pointer to this thread's newlib state + struct _reent* reent; + + // Pointer to this thread's thread-local segment + void* tls_tp; // !! Offset needs to be TLS+0x1F8 for __aarch64_read_tp !! +} ThreadVars; + +static inline ThreadVars* getThreadVars(void) { + return (ThreadVars*)((u8*)armGetTls() + 0x1E0); +} +// end copy + + +/* + * Initialization. + */ +static +void _noop(void) +{ + // printf("_noop function called\n"); +} + + +static void +PyThread__init_thread(void) +{ + Thread thread1; + threadCreate(&thread1, (void *) _noop, NULL, THREAD_STACK_SIZE, THREAD_PRIORITY, THREAD_CPU_ID); + threadStart(&thread1); + threadWaitForExit(&thread1); + threadClose(&thread1); +} + +/* + * Thread support. + */ + +typedef struct { + ThreadFunc func; + void * arg; +} _wrapperArgs; + +static +void _wrapperFunc(_wrapperArgs * args) { + // printf("_wrapperFunc called -> %p, %p\n", args->func, args->arg); + args->func(args->arg); +} + +long +PyThread_start_new_thread(void (*func)(void *), void *arg) +{ + dprintf(("PyThread_start_new_thread called\n")); + if (!initialized) + PyThread_init_thread(); + + _wrapperArgs wargs; + wargs.arg = arg; + wargs.func = func; + + Thread * t = (Thread *)PyMem_RawMalloc(sizeof(Thread)); + int rc = threadCreate(t, (ThreadFunc)_wrapperFunc, &wargs, THREAD_STACK_SIZE, THREAD_PRIORITY, THREAD_CPU_ID); + // printf("PyThread thread created -> %d\n", rc); + if (rc < 0) return -1; + rc = threadStart(t); + // printf("PyThread thread started -> %p, %d\n", t, rc); + + return rc < 0 ? -1 : 0; +} + +long +PyThread_get_thread_ident(void) +{ + if (!initialized) + PyThread_init_thread(); + Thread * t = getThreadVars()->thread_ptr; + // if (t != NULL) printf("PyThread_get_thread_ident called -> %p\n", t); + return (long)t + 1; +} + +void +PyThread_exit_thread(void) +{ + dprintf(("PyThread_exit_thread called\n")); + if (!initialized) + exit(0); + + // TODO: fix this + // Thread * t = getThreadVars()->thread_ptr; + // threadClose(t); + // PyMem_RawFree((void *)t); + svcExitThread(); +} + +/* + * Lock support. + */ + +typedef struct { + char locked; + CondVar cv; + Mutex mutex; +} _thread_lock; + +PyThread_type_lock +PyThread_allocate_lock(void) +{ + dprintf(("PyThread_allocate_lock called\n")); + if (!initialized) + PyThread_init_thread(); + + _thread_lock * lock = (_thread_lock *)PyMem_RawMalloc(sizeof(_thread_lock)); + lock->locked = 0; + + mutexInit(&(lock->mutex)); + + condvarInit(&(lock->cv), &(lock->mutex)); + + dprintf(("PyThread_allocate_lock() -> %p\n", lock)); + return (PyThread_type_lock) lock; +} + +void +PyThread_free_lock(PyThread_type_lock lock) +{ + _thread_lock * thelock = (_thread_lock *)lock; + + + dprintf(("PyThread_free_lock(%p) called\n", lock)); + + if (!thelock) + return; + + PyMem_RawFree((void *)thelock); +} + +int +PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) +{ + return PyThread_acquire_lock_timed(lock, waitflag ? -1 : 0, 0); +} + +PyLockStatus +PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, + int intr_flag) +{ + PyLockStatus success; + int status; + _thread_lock * thelock = (_thread_lock *)lock; + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) called\n", + lock, microseconds, intr_flag)); + + if (thelock->locked == 1 && microseconds == 0) { + success = PY_LOCK_FAILURE; + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", + lock, microseconds, intr_flag, success)); + return success; + } + + mutexLock(&(thelock->mutex)); + if (thelock->locked == 0) { + success = PY_LOCK_ACQUIRED; + } else if (microseconds == 0) { + success = PY_LOCK_FAILURE; + } else { + u64 ns = microseconds * 1000; + + success = PY_LOCK_FAILURE; + while (success == PY_LOCK_FAILURE) { + if (microseconds > 0) { + status = condvarWaitTimeout(&(thelock->cv), ns); + if (status == 0xEA01) // on timeout + break; + } + else { + status = condvarWait(&(thelock->cv)); + } + + if (intr_flag && status == 0 && thelock->locked) { + /* We were woken up, but didn't get the lock. We probably received + * a signal. Return PY_LOCK_INTR to allow the caller to handle + * it and retry. */ + success = PY_LOCK_INTR; + break; + } else if (!thelock->locked) { + success = PY_LOCK_ACQUIRED; + } else { + success = PY_LOCK_FAILURE; + } + } + } + + if (success == PY_LOCK_ACQUIRED) { + thelock->locked = 1; + } + mutexUnlock(&(thelock->mutex)); + + dprintf(("PyThread_acquire_lock_timed(%p, %lld, %d) -> %d\n", + lock, microseconds, intr_flag, success)); + return success; +} + +void +PyThread_release_lock(PyThread_type_lock lock) +{ + _thread_lock * thelock = (_thread_lock *)lock; + dprintf(("PyThread_release_lock(%p) called\n", lock)); + // printf("PyThread_release_lock(%p) called\n", lock); + + mutexLock(&(thelock->mutex)); + dprintf(("PyThread_release_lock(%p) mutex locked\n", lock)); + thelock->locked = 0; + condvarWakeOne(&(thelock->cv)); + mutexUnlock(&(thelock->mutex)); +} + +/* The following are only needed if native TLS support exists */ +// #define Py_HAVE_NATIVE_TLS + +#ifdef Py_HAVE_NATIVE_TLS +int +PyThread_create_key(void) +{ + int result; + return result; +} + +void +PyThread_delete_key(int key) +{ + +} + +int +PyThread_set_key_value(int key, void *value) +{ + int ok; + + /* A failure in this case returns -1 */ + if (!ok) + return -1; + return 0; +} + +void * +PyThread_get_key_value(int key) +{ + void *result; + + return result; +} + +void +PyThread_delete_key_value(int key) +{ + +} + +void +PyThread_ReInitTLS(void) +{ + +} + +#endif From a290ef780577745c01d20ef3a1253a044e070d27 Mon Sep 17 00:00:00 2001 From: Elnard Utiushev Date: Mon, 16 Jul 2018 18:52:55 +0600 Subject: [PATCH 52/72] Enable zlib (#6) --- Makefile | 2 +- python_build/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0ff91e0..eee31ea 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions -std=gnu++11 ASFLAGS := -g $(ARCH) LDFLAGS = -specs=$(DEVKITPRO)/libnx/switch.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map) -LIBS := -lpython3.5 -lm -lnx +LIBS := -lpython3.5 -lm -lz -lnx #--------------------------------------------------------------------------------- # list of directories containing libraries, this must be the top level containing diff --git a/python_build/Makefile b/python_build/Makefile index 0d6aeda..6357178 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -126,7 +126,7 @@ patchPY: cloneNX echo ac_cv_file__dev_ptc=no >>$(PYDIR)/config.site echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old - cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c hashtable.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" | $(SED) "s/_tracemalloc/# _tracemalloc/" >$(PYDIR)/Modules/Setup.dist + cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c hashtable.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\|zlib\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" | $(SED) "s/_tracemalloc/# _tracemalloc/" >$(PYDIR)/Modules/Setup.dist cp -r _nx-*/_nx $(PYDIR)/Modules/ touch patchPY From dfbe428d856e2f111f61334bd1c7331f9c579f48 Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 18 Jul 2018 21:14:38 +0600 Subject: [PATCH 53/72] Bump version to v0.5.2-alpha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eee31ea..7304bff 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := $(TARGET) APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.5.1-alpha +APP_VERSION := 0.5.2-alpha #--------------------------------------------------------------------------------- # options for code generation From db758d6a61fe84d2e2ac40435110722b1eaf6fb0 Mon Sep 17 00:00:00 2001 From: elnardu Date: Sat, 4 Aug 2018 21:34:38 +0600 Subject: [PATCH 54/72] Switch to libnx's getaddrinfo --- python_config/pyconfig.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_config/pyconfig.h b/python_config/pyconfig.h index 0f9c0c6..80031f6 100644 --- a/python_config/pyconfig.h +++ b/python_config/pyconfig.h @@ -335,7 +335,7 @@ #define HAVE_GCC_UINT128_T 1 /* Define if you have the getaddrinfo function. */ -#undef HAVE_GETADDRINFO +#define HAVE_GETADDRINFO // TODO clarify what these are #define SOCK_RAW 3 From 87bbaef9c7f21c4ebf110c3a629fbcc2efd3f87e Mon Sep 17 00:00:00 2001 From: elnardu Date: Sat, 4 Aug 2018 21:47:16 +0600 Subject: [PATCH 55/72] Delete old getaddrinfo hack --- python_build/Makefile | 1 - python_config/getaddrinfo.c | 665 ------------------------------------ 2 files changed, 666 deletions(-) delete mode 100644 python_config/getaddrinfo.c diff --git a/python_build/Makefile b/python_build/Makefile index 6357178..0c49e5e 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -103,7 +103,6 @@ soospatchPY: compilePY cp $(PYCONFIG_DIR)/condvar.h $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/pylifecycle.c $(PYDIR)/Python/ cp $(PYCONFIG_DIR)/ceval_gil.h $(PYDIR)/Python/ - cp $(PYCONFIG_DIR)/getaddrinfo.c $(PYDIR)/Modules/ cp $(PYDIR)/Modules/posixmodule.c $(PYDIR)/Modules/posixmodule.c_old cat $(PYDIR)/Modules/posixmodule.c_old | $(SED) 's/return utime(path, time)/errno=ENOENT; return -1/g' | $(SED) 's/define LSTAT lstat/define LSTAT stat/' | $(SED) '1s/^/#include /' | $(SED) 's/access(path->narrow, mode)/1/' | $(SED) 's/\(^[^rt]*time_t atime, mtime;.*\)/return NULL; \1/' | $(SED) 's/\(^[^ri]*int i = (int)umask(mask);.*\)/int i=0; return NULL;/' | $(SED) 's/^\([^#][^#]*#undef HAVE_FSTATVFS.*\)/\#undef HAVE_FSTATVFS \1/' | $(SED) 's/#define HAVE_\(EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/g' | $(SED) 's/^#define HAVE_\(STATVFS\|SYS_STATVFS_H\|FDATASYNC\|FTIME\|SYMLINK\|EXECV\|FORK\|GETEGID\|GETEUID\|GETGID\|GETPPID\|GETUID\|KILL\|PIPE\|POPEN\|SYSTEM\|TTYNAME\|SYMLINK\|UTIME_H\|FDATASYNC\).*/#undef HAVE_\1/' >$(PYDIR)/Modules/posixmodule.c cp $(PYDIR)/Modules/socketmodule.c $(PYDIR)/Modules/socketmodule.c_old diff --git a/python_config/getaddrinfo.c b/python_config/getaddrinfo.c deleted file mode 100644 index ed40977..0000000 --- a/python_config/getaddrinfo.c +++ /dev/null @@ -1,665 +0,0 @@ -/* - * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the project nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * "#ifdef FAITH" part is local hack for supporting IPv4-v6 translator. - * - * Issues to be discussed: - * - Thread safe-ness must be checked. - * - Return values. There are nonstandard return values defined and used - * in the source code. This is because RFC2133 is silent about which error - * code must be returned for which situation. - * - PF_UNSPEC case would be handled in getipnodebyname() with the AI_ALL flag. - */ - -#if 0 -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "addrinfo.h" -#endif - -#if defined(__KAME__) && defined(ENABLE_IPV6) -# define FAITH -#endif - -#define SUCCESS 0 -#define GAI_ANY 0 -#define YES 1 -#define NO 0 - -#ifdef FAITH -static int translate = NO; -static struct in6_addr faith_prefix = IN6ADDR_GAI_ANY_INIT; -#endif - -static const char in_addrany[] = { 0, 0, 0, 0 }; -static const char in6_addrany[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; -static const char in_loopback[] = { 127, 0, 0, 1 }; -static const char in6_loopback[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 -}; - -struct sockinet { - u_char si_len; - u_char si_family; - u_short si_port; -}; - -static struct gai_afd { - int a_af; - int a_addrlen; - int a_socklen; - int a_off; - const char *a_addrany; - const char *a_loopback; -} gai_afdl [] = { -#ifdef ENABLE_IPV6 -#define N_INET6 0 - {PF_INET6, sizeof(struct in6_addr), - sizeof(struct sockaddr_in6), - offsetof(struct sockaddr_in6, sin6_addr), - in6_addrany, in6_loopback}, -#define N_INET 1 -#else -#define N_INET 0 -#endif - {PF_INET, sizeof(struct in_addr), - sizeof(struct sockaddr_in), - offsetof(struct sockaddr_in, sin_addr), - in_addrany, in_loopback}, - {0, 0, 0, 0, NULL, NULL}, -}; - -#ifdef ENABLE_IPV6 -#define PTON_MAX 16 -#else -#define PTON_MAX 4 -#endif - -#ifndef IN_MULTICAST -#define IN_MULTICAST(i) (((i) & 0xf0000000U) == 0xe0000000U) -#endif - -#ifndef IN_EXPERIMENTAL -#define IN_EXPERIMENTAL(i) (((i) & 0xe0000000U) == 0xe0000000U) -#endif - -#ifndef IN_LOOPBACKNET -#define IN_LOOPBACKNET 127 -#endif - -static int get_name(const char *, struct gai_afd *, - struct addrinfo **, char *, struct addrinfo *, - int); -static int get_addr(const char *, int, struct addrinfo **, - struct addrinfo *, int); -static int str_isnumber(const char *); - -static char *ai_errlist[] = { - "success.", - "address family for hostname not supported.", /* EAI_ADDRFAMILY */ - "temporary failure in name resolution.", /* EAI_AGAIN */ - "invalid value for ai_flags.", /* EAI_BADFLAGS */ - "non-recoverable failure in name resolution.", /* EAI_FAIL */ - "ai_family not supported.", /* EAI_FAMILY */ - "memory allocation failure.", /* EAI_MEMORY */ - "no address associated with hostname.", /* EAI_NODATA */ - "hostname nor servname provided, or not known.",/* EAI_NONAME */ - "servname not supported for ai_socktype.", /* EAI_SERVICE */ - "ai_socktype not supported.", /* EAI_SOCKTYPE */ - "system error returned in errno.", /* EAI_SYSTEM */ - "invalid value for hints.", /* EAI_BADHINTS */ - "resolved protocol is unknown.", /* EAI_PROTOCOL */ - "unknown error.", /* EAI_MAX */ -}; - -#define GET_CANONNAME(ai, str) \ -if (pai->ai_flags & AI_CANONNAME) {\ - if (((ai)->ai_canonname = (char *)malloc(strlen(str) + 1)) != NULL) {\ - strcpy((ai)->ai_canonname, (str));\ - } else {\ - error = EAI_MEMORY;\ - goto free;\ - }\ -} - -#ifdef HAVE_SOCKADDR_SA_LEN -#define GET_AI(ai, gai_afd, addr, port) {\ - char *p;\ - if (((ai) = (struct addrinfo *)malloc(sizeof(struct addrinfo) +\ - ((gai_afd)->a_socklen)))\ - == NULL) goto free;\ - memcpy(ai, pai, sizeof(struct addrinfo));\ - (ai)->ai_addr = (struct sockaddr *)((ai) + 1);\ - memset((ai)->ai_addr, 0, (gai_afd)->a_socklen);\ - (ai)->ai_addr->sa_len = (ai)->ai_addrlen = (gai_afd)->a_socklen;\ - (ai)->ai_addr->sa_family = (ai)->ai_family = (gai_afd)->a_af;\ - ((struct sockinet *)(ai)->ai_addr)->si_port = port;\ - p = (char *)((ai)->ai_addr);\ - memcpy(p + (gai_afd)->a_off, (addr), (gai_afd)->a_addrlen);\ -} -#else -#define GET_AI(ai, gai_afd, addr, port) {\ - char *p;\ - if (((ai) = (struct addrinfo *)malloc(sizeof(struct addrinfo) +\ - ((gai_afd)->a_socklen)))\ - == NULL) goto free;\ - memcpy(ai, pai, sizeof(struct addrinfo));\ - (ai)->ai_addr = (struct sockaddr *)((ai) + 1);\ - memset((ai)->ai_addr, 0, (gai_afd)->a_socklen);\ - (ai)->ai_addrlen = (gai_afd)->a_socklen;\ - (ai)->ai_addr->sa_family = (ai)->ai_family = (gai_afd)->a_af;\ - ((struct sockinet *)(ai)->ai_addr)->si_port = port;\ - p = (char *)((ai)->ai_addr);\ - memcpy(p + (gai_afd)->a_off, (addr), (gai_afd)->a_addrlen);\ -} -#endif - -#define ERR(err) { error = (err); goto bad; } - -char * -gai_strerror(int ecode) -{ - if (ecode < 0 || ecode > EAI_MAX) - ecode = EAI_MAX; - return ai_errlist[ecode]; -} - -void -freeaddrinfo(struct addrinfo *ai) -{ - struct addrinfo *next; - - do { - next = ai->ai_next; - if (ai->ai_canonname) - free(ai->ai_canonname); - /* no need to free(ai->ai_addr) */ - free(ai); - } while ((ai = next) != NULL); -} - -static int -str_isnumber(const char *p) -{ - unsigned char *q = (unsigned char *)p; - while (*q) { - if (! isdigit(*q)) - return NO; - q++; - } - return YES; -} - -int -getaddrinfo(const char*hostname, const char*servname, - const struct addrinfo *hints, struct addrinfo **res) -{ - struct addrinfo sentinel; - struct addrinfo *top = NULL; - struct addrinfo *cur; - int i, error = 0; - char pton[PTON_MAX]; - struct addrinfo ai; - struct addrinfo *pai; - u_short port; - -#ifdef FAITH - static int firsttime = 1; - - if (firsttime) { - /* translator hack */ - { - char *q = getenv("GAI"); - if (q && inet_pton(AF_INET6, q, &faith_prefix) == 1) - translate = YES; - } - firsttime = 0; - } -#endif - - /* initialize file static vars */ - sentinel.ai_next = NULL; - cur = &sentinel; - pai = &ai; - pai->ai_flags = 0; - pai->ai_family = PF_UNSPEC; - pai->ai_socktype = GAI_ANY; - pai->ai_protocol = GAI_ANY; - pai->ai_addrlen = 0; - pai->ai_canonname = NULL; - pai->ai_addr = NULL; - pai->ai_next = NULL; - port = GAI_ANY; - - if (hostname == NULL && servname == NULL) - return EAI_NONAME; - if (hints) { - /* error check for hints */ - if (hints->ai_addrlen || hints->ai_canonname || - hints->ai_addr || hints->ai_next) - ERR(EAI_BADHINTS); /* xxx */ - if (hints->ai_flags & ~AI_MASK) - ERR(EAI_BADFLAGS); - switch (hints->ai_family) { - case PF_UNSPEC: - case PF_INET: -#ifdef ENABLE_IPV6 - case PF_INET6: -#endif - break; - default: - ERR(EAI_FAMILY); - } - memcpy(pai, hints, sizeof(*pai)); - switch (pai->ai_socktype) { - case GAI_ANY: - switch (pai->ai_protocol) { - case GAI_ANY: - break; - case IPPROTO_UDP: - pai->ai_socktype = SOCK_DGRAM; - break; - case IPPROTO_TCP: - pai->ai_socktype = SOCK_STREAM; - break; - default: - pai->ai_socktype = SOCK_RAW; - break; - } - break; - case SOCK_RAW: - break; - case SOCK_DGRAM: - if (pai->ai_protocol != IPPROTO_UDP && - pai->ai_protocol != GAI_ANY) - ERR(EAI_BADHINTS); /*xxx*/ - pai->ai_protocol = IPPROTO_UDP; - break; - case SOCK_STREAM: - if (pai->ai_protocol != IPPROTO_TCP && - pai->ai_protocol != GAI_ANY) - ERR(EAI_BADHINTS); /*xxx*/ - pai->ai_protocol = IPPROTO_TCP; - break; - default: - ERR(EAI_SOCKTYPE); - /* unreachable */ - } - } - - /* - * service port - */ - if (servname) { - if (str_isnumber(servname)) { - if (pai->ai_socktype == GAI_ANY) { - /* caller accept *GAI_ANY* socktype */ - pai->ai_socktype = SOCK_DGRAM; - pai->ai_protocol = IPPROTO_UDP; - } - port = htons((u_short)atoi(servname)); - } else { - struct servent *sp; - char *proto; - - proto = NULL; - switch (pai->ai_socktype) { - case GAI_ANY: - proto = NULL; - break; - case SOCK_DGRAM: - proto = "udp"; - break; - case SOCK_STREAM: - proto = "tcp"; - break; - default: - fprintf(stderr, "panic!\n"); - break; - } - if ((sp = getservbyname(servname, proto)) == NULL) - ERR(EAI_SERVICE); - port = sp->s_port; - if (pai->ai_socktype == GAI_ANY) { - if (strcmp(sp->s_proto, "udp") == 0) { - pai->ai_socktype = SOCK_DGRAM; - pai->ai_protocol = IPPROTO_UDP; - } else if (strcmp(sp->s_proto, "tcp") == 0) { - pai->ai_socktype = SOCK_STREAM; - pai->ai_protocol = IPPROTO_TCP; - } else - ERR(EAI_PROTOCOL); /*xxx*/ - } - } - } - - /* - * hostname == NULL. - * passive socket -> anyaddr (0.0.0.0 or ::) - * non-passive socket -> localhost (127.0.0.1 or ::1) - */ - if (hostname == NULL) { - struct gai_afd *gai_afd; - - for (gai_afd = &gai_afdl[0]; gai_afd->a_af; gai_afd++) { - if (!(pai->ai_family == PF_UNSPEC - || pai->ai_family == gai_afd->a_af)) { - continue; - } - - if (pai->ai_flags & AI_PASSIVE) { - GET_AI(cur->ai_next, gai_afd, gai_afd->a_addrany, port); - /* xxx meaningless? - * GET_CANONNAME(cur->ai_next, "anyaddr"); - */ - } else { - GET_AI(cur->ai_next, gai_afd, gai_afd->a_loopback, - port); - /* xxx meaningless? - * GET_CANONNAME(cur->ai_next, "localhost"); - */ - } - cur = cur->ai_next; - } - top = sentinel.ai_next; - if (top) - goto good; - else - ERR(EAI_FAMILY); - } - - /* hostname as numeric name */ - for (i = 0; gai_afdl[i].a_af; i++) { - if (inet_pton(gai_afdl[i].a_af, hostname, pton)) { - u_long v4a; -#ifdef ENABLE_IPV6 - u_char pfx; -#endif - - switch (gai_afdl[i].a_af) { - case AF_INET: - v4a = ((struct in_addr *)pton)->s_addr; - v4a = ntohl(v4a); - if (IN_MULTICAST(v4a) || IN_EXPERIMENTAL(v4a)) - pai->ai_flags &= ~AI_CANONNAME; - v4a >>= IN_CLASSA_NSHIFT; - if (v4a == 0 || v4a == IN_LOOPBACKNET) - pai->ai_flags &= ~AI_CANONNAME; - break; -#ifdef ENABLE_IPV6 - case AF_INET6: - pfx = ((struct in6_addr *)pton)->s6_addr[0]; - if (pfx == 0 || pfx == 0xfe || pfx == 0xff) - pai->ai_flags &= ~AI_CANONNAME; - break; -#endif - } - - if (pai->ai_family == gai_afdl[i].a_af || - pai->ai_family == PF_UNSPEC) { - if (! (pai->ai_flags & AI_CANONNAME)) { - GET_AI(top, &gai_afdl[i], pton, port); - goto good; - } - /* - * if AI_CANONNAME and if reverse lookup - * fail, return ai anyway to pacify - * calling application. - * - * XXX getaddrinfo() is a name->address - * translation function, and it looks strange - * that we do addr->name translation here. - */ - get_name(pton, &gai_afdl[i], &top, pton, pai, port); - goto good; - } else - ERR(EAI_FAMILY); /*xxx*/ - } - } - - if (pai->ai_flags & AI_NUMERICHOST) - ERR(EAI_NONAME); - - /* hostname as alphabetical name */ - error = get_addr(hostname, pai->ai_family, &top, pai, port); - if (error == 0) { - if (top) { - good: - *res = top; - return SUCCESS; - } else - error = EAI_FAIL; - } - free: - if (top) - freeaddrinfo(top); - bad: - *res = NULL; - return error; -} - -static int -get_name(addr, gai_afd, res, numaddr, pai, port0) - const char *addr; - struct gai_afd *gai_afd; - struct addrinfo **res; - char *numaddr; - struct addrinfo *pai; - int port0; -{ - u_short port = port0 & 0xffff; - struct hostent *hp; - struct addrinfo *cur; - int error = 0; -#ifdef ENABLE_IPV6 - int h_error; -#endif - -#ifdef ENABLE_IPV6 - hp = getipnodebyaddr(addr, gai_afd->a_addrlen, gai_afd->a_af, &h_error); -#else - hp = gethostbyaddr(addr, gai_afd->a_addrlen, AF_INET); -#endif - if (hp && hp->h_name && hp->h_name[0] && hp->h_addr_list[0]) { - GET_AI(cur, gai_afd, hp->h_addr_list[0], port); - GET_CANONNAME(cur, hp->h_name); - } else - GET_AI(cur, gai_afd, numaddr, port); - -#ifdef ENABLE_IPV6 - if (hp) - freehostent(hp); -#endif - *res = cur; - return SUCCESS; - free: - if (cur) - freeaddrinfo(cur); -#ifdef ENABLE_IPV6 - if (hp) - freehostent(hp); -#endif - /* bad: */ - *res = NULL; - return error; -} - -static int -get_addr(hostname, af, res, pai, port0) - const char *hostname; - int af; - struct addrinfo **res; - struct addrinfo *pai; - int port0; -{ - u_short port = port0 & 0xffff; - struct addrinfo sentinel; - struct hostent *hp; - struct addrinfo *top, *cur; - struct gai_afd *gai_afd; - int i, error = 0, h_error; - char *ap; - - top = NULL; - sentinel.ai_next = NULL; - cur = &sentinel; - -#ifdef SWITCH // XXX dirty hack XXX - struct sockaddr_in *saddr; - - if (!(af == AF_INET || af == AF_UNSPEC)) - return EAI_NODATA; - - saddr = calloc(1, sizeof(*saddr)); - saddr->sin_family = AF_INET; - saddr->sin_port = port0; - - /* XXX libnx inet_pton is currently broken, we will ignore the return value for now... */ - if (inet_pton(AF_INET, hostname, &(saddr->sin_addr)) != 1 && 0) { - free(saddr); - error = EAI_NODATA; - goto free; - } - - top = calloc(1, sizeof(*top)); - top->ai_family = AF_INET; - top->ai_addrlen = sizeof(*saddr); - top->ai_addr = (struct sockaddr *) saddr; - - *res = top; - return SUCCESS; -#endif - -#ifdef ENABLE_IPV6 - if (af == AF_UNSPEC) { - hp = getipnodebyname(hostname, AF_INET6, - AI_ADDRCONFIG|AI_ALL|AI_V4MAPPED, &h_error); - } else - hp = getipnodebyname(hostname, af, AI_ADDRCONFIG, &h_error); -#else - hp = gethostbyname(hostname); - h_error = h_errno; -#endif - if (hp == NULL) { - switch (h_error) { - case HOST_NOT_FOUND: - case NO_DATA: - error = EAI_NODATA; - break; - case TRY_AGAIN: - error = EAI_AGAIN; - break; - case NO_RECOVERY: - default: - error = EAI_FAIL; - break; - } - goto free; - } - - if ((hp->h_name == NULL) || (hp->h_name[0] == 0) || - (hp->h_addr_list[0] == NULL)) { - error = EAI_FAIL; - goto free; - } - - for (i = 0; (ap = hp->h_addr_list[i]) != NULL; i++) { - switch (af) { -#ifdef ENABLE_IPV6 - case AF_INET6: - gai_afd = &gai_afdl[N_INET6]; - break; -#endif -#ifndef ENABLE_IPV6 - default: /* AF_UNSPEC */ -#endif - case AF_INET: - gai_afd = &gai_afdl[N_INET]; - break; -#ifdef ENABLE_IPV6 - default: /* AF_UNSPEC */ - if (IN6_IS_ADDR_V4MAPPED((struct in6_addr *)ap)) { - ap += sizeof(struct in6_addr) - - sizeof(struct in_addr); - gai_afd = &gai_afdl[N_INET]; - } else - gai_afd = &gai_afdl[N_INET6]; - break; -#endif - } -#ifdef FAITH - if (translate && gai_afd->a_af == AF_INET) { - struct in6_addr *in6; - - GET_AI(cur->ai_next, &gai_afdl[N_INET6], ap, port); - in6 = &((struct sockaddr_in6 *)cur->ai_next->ai_addr)->sin6_addr; - memcpy(&in6->s6_addr32[0], &faith_prefix, - sizeof(struct in6_addr) - sizeof(struct in_addr)); - memcpy(&in6->s6_addr32[3], ap, sizeof(struct in_addr)); - } else -#endif /* FAITH */ - GET_AI(cur->ai_next, gai_afd, ap, port); - if (cur == &sentinel) { - top = cur->ai_next; - GET_CANONNAME(top, hp->h_name); - } - cur = cur->ai_next; - } -#ifdef ENABLE_IPV6 - freehostent(hp); -#endif - *res = top; - return SUCCESS; - free: - if (top) - freeaddrinfo(top); -#ifdef ENABLE_IPV6 - if (hp) - freehostent(hp); -#endif -/* bad: */ - *res = NULL; - return error; -} From 76d635c3681d4b438840cc306d05f6d409f4df24 Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 8 Aug 2018 11:36:06 +0600 Subject: [PATCH 56/72] Use cxx linker --- Makefile | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 7304bff..ef6044f 100644 --- a/Makefile +++ b/Makefile @@ -88,19 +88,12 @@ SFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.s))) BINFILES := $(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*))) #--------------------------------------------------------------------------------- -# use CXX for linking C++ projects, CC for standard C +# use CXX for linking C++ projects #--------------------------------------------------------------------------------- -ifeq ($(strip $(CPPFILES)),) -#--------------------------------------------------------------------------------- - export LD := $(CC) -#--------------------------------------------------------------------------------- -else -#--------------------------------------------------------------------------------- - export LD := $(CXX) -#--------------------------------------------------------------------------------- -endif +export LD := $(CXX) #--------------------------------------------------------------------------------- + export OFILES := $(addsuffix .o,$(BINFILES)) \ $(CPPFILES:.cpp=.o) $(CFILES:.c=.o) $(SFILES:.s=.o) From 7161e911ffb155492174c3ff8c7fa96e8ad30e98 Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 8 Aug 2018 11:40:52 +0600 Subject: [PATCH 57/72] Patches necessary for building cpp python modules --- python_build/Makefile | 4 + python_config/pyatomic.h | 250 +++++++++++++++++++++++++++++++++++ python_config/pystate.h | 279 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 533 insertions(+) create mode 100644 python_config/pyatomic.h create mode 100644 python_config/pystate.h diff --git a/python_build/Makefile b/python_build/Makefile index 0c49e5e..7a36cdc 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -93,6 +93,10 @@ cloneNX: touch cloneNX soospatchPY: compilePY + # See https://bugs.python.org/issue23644 + cp $(PYCONFIG_DIR)/pyatomic.h $(PYDIR)/Include + cp $(PYCONFIG_DIR)/pystate.h $(PYDIR)/Include + cp $(PYCONFIG_DIR)/pyconfig.h $(PYDIR)/ cp $(PYCONFIG_DIR)/intrcheck.c $(PYDIR)/Parser/ cp $(PYCONFIG_DIR)/pytime.c $(PYDIR)/Python/ diff --git a/python_config/pyatomic.h b/python_config/pyatomic.h new file mode 100644 index 0000000..527aeb8 --- /dev/null +++ b/python_config/pyatomic.h @@ -0,0 +1,250 @@ +#ifndef Py_ATOMIC_H +#define Py_ATOMIC_H +#if defined(Py_BUILD_CORE) && !defined(__cplusplus) + +#include "dynamic_annotations.h" + +#include "pyconfig.h" + +#if defined(HAVE_STD_ATOMIC) +#include +#endif + +/* This is modeled after the atomics interface from C1x, according to + * the draft at + * http://www.open-std.org/JTC1/SC22/wg14/www/docs/n1425.pdf. + * Operations and types are named the same except with a _Py_ prefix + * and have the same semantics. + * + * Beware, the implementations here are deep magic. + */ + +#if defined(HAVE_STD_ATOMIC) + +typedef enum _Py_memory_order { + _Py_memory_order_relaxed = memory_order_relaxed, + _Py_memory_order_acquire = memory_order_acquire, + _Py_memory_order_release = memory_order_release, + _Py_memory_order_acq_rel = memory_order_acq_rel, + _Py_memory_order_seq_cst = memory_order_seq_cst +} _Py_memory_order; + +typedef struct _Py_atomic_address { + atomic_uintptr_t _value; +} _Py_atomic_address; + +typedef struct _Py_atomic_int { + atomic_int _value; +} _Py_atomic_int; + +#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ + atomic_signal_fence(ORDER) + +#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ + atomic_thread_fence(ORDER) + +#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ + atomic_store_explicit(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER) + +#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ + atomic_load_explicit(&(ATOMIC_VAL)->_value, ORDER) + +/* Use builtin atomic operations in GCC >= 4.7 */ +#elif defined(HAVE_BUILTIN_ATOMIC) + +typedef enum _Py_memory_order { + _Py_memory_order_relaxed = __ATOMIC_RELAXED, + _Py_memory_order_acquire = __ATOMIC_ACQUIRE, + _Py_memory_order_release = __ATOMIC_RELEASE, + _Py_memory_order_acq_rel = __ATOMIC_ACQ_REL, + _Py_memory_order_seq_cst = __ATOMIC_SEQ_CST +} _Py_memory_order; + +typedef struct _Py_atomic_address { + Py_uintptr_t _value; +} _Py_atomic_address; + +typedef struct _Py_atomic_int { + int _value; +} _Py_atomic_int; + +#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) \ + __atomic_signal_fence(ORDER) + +#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) \ + __atomic_thread_fence(ORDER) + +#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ + (assert((ORDER) == __ATOMIC_RELAXED \ + || (ORDER) == __ATOMIC_SEQ_CST \ + || (ORDER) == __ATOMIC_RELEASE), \ + __atomic_store_n(&(ATOMIC_VAL)->_value, NEW_VAL, ORDER)) + +#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ + (assert((ORDER) == __ATOMIC_RELAXED \ + || (ORDER) == __ATOMIC_SEQ_CST \ + || (ORDER) == __ATOMIC_ACQUIRE \ + || (ORDER) == __ATOMIC_CONSUME), \ + __atomic_load_n(&(ATOMIC_VAL)->_value, ORDER)) + +#else + +typedef enum _Py_memory_order { + _Py_memory_order_relaxed, + _Py_memory_order_acquire, + _Py_memory_order_release, + _Py_memory_order_acq_rel, + _Py_memory_order_seq_cst +} _Py_memory_order; + +typedef struct _Py_atomic_address { + Py_uintptr_t _value; +} _Py_atomic_address; + +typedef struct _Py_atomic_int { + int _value; +} _Py_atomic_int; + +/* Only support GCC (for expression statements) and x86 (for simple + * atomic semantics) for now */ +#if defined(__GNUC__) && (defined(__i386__) || defined(__amd64)) + +static __inline__ void +_Py_atomic_signal_fence(_Py_memory_order order) +{ + if (order != _Py_memory_order_relaxed) + __asm__ volatile("":::"memory"); +} + +static __inline__ void +_Py_atomic_thread_fence(_Py_memory_order order) +{ + if (order != _Py_memory_order_relaxed) + __asm__ volatile("mfence":::"memory"); +} + +/* Tell the race checker about this operation's effects. */ +static __inline__ void +_Py_ANNOTATE_MEMORY_ORDER(const volatile void *address, _Py_memory_order order) +{ + (void)address; /* shut up -Wunused-parameter */ + switch(order) { + case _Py_memory_order_release: + case _Py_memory_order_acq_rel: + case _Py_memory_order_seq_cst: + _Py_ANNOTATE_HAPPENS_BEFORE(address); + break; + case _Py_memory_order_relaxed: + case _Py_memory_order_acquire: + break; + } + switch(order) { + case _Py_memory_order_acquire: + case _Py_memory_order_acq_rel: + case _Py_memory_order_seq_cst: + _Py_ANNOTATE_HAPPENS_AFTER(address); + break; + case _Py_memory_order_relaxed: + case _Py_memory_order_release: + break; + } +} + +#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ + __extension__ ({ \ + __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ + __typeof__(atomic_val->_value) new_val = NEW_VAL;\ + volatile __typeof__(new_val) *volatile_data = &atomic_val->_value; \ + _Py_memory_order order = ORDER; \ + _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ + \ + /* Perform the operation. */ \ + _Py_ANNOTATE_IGNORE_WRITES_BEGIN(); \ + switch(order) { \ + case _Py_memory_order_release: \ + _Py_atomic_signal_fence(_Py_memory_order_release); \ + /* fallthrough */ \ + case _Py_memory_order_relaxed: \ + *volatile_data = new_val; \ + break; \ + \ + case _Py_memory_order_acquire: \ + case _Py_memory_order_acq_rel: \ + case _Py_memory_order_seq_cst: \ + __asm__ volatile("xchg %0, %1" \ + : "+r"(new_val) \ + : "m"(atomic_val->_value) \ + : "memory"); \ + break; \ + } \ + _Py_ANNOTATE_IGNORE_WRITES_END(); \ + }) + +#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ + __extension__ ({ \ + __typeof__(ATOMIC_VAL) atomic_val = ATOMIC_VAL; \ + __typeof__(atomic_val->_value) result; \ + volatile __typeof__(result) *volatile_data = &atomic_val->_value; \ + _Py_memory_order order = ORDER; \ + _Py_ANNOTATE_MEMORY_ORDER(atomic_val, order); \ + \ + /* Perform the operation. */ \ + _Py_ANNOTATE_IGNORE_READS_BEGIN(); \ + switch(order) { \ + case _Py_memory_order_release: \ + case _Py_memory_order_acq_rel: \ + case _Py_memory_order_seq_cst: \ + /* Loads on x86 are not releases by default, so need a */ \ + /* thread fence. */ \ + _Py_atomic_thread_fence(_Py_memory_order_release); \ + break; \ + default: \ + /* No fence */ \ + break; \ + } \ + result = *volatile_data; \ + switch(order) { \ + case _Py_memory_order_acquire: \ + case _Py_memory_order_acq_rel: \ + case _Py_memory_order_seq_cst: \ + /* Loads on x86 are automatically acquire operations so */ \ + /* can get by with just a compiler fence. */ \ + _Py_atomic_signal_fence(_Py_memory_order_acquire); \ + break; \ + default: \ + /* No fence */ \ + break; \ + } \ + _Py_ANNOTATE_IGNORE_READS_END(); \ + result; \ + }) + +#else /* !gcc x86 */ +/* Fall back to other compilers and processors by assuming that simple + volatile accesses are atomic. This is false, so people should port + this. */ +#define _Py_atomic_signal_fence(/*memory_order*/ ORDER) ((void)0) +#define _Py_atomic_thread_fence(/*memory_order*/ ORDER) ((void)0) +#define _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, ORDER) \ + ((ATOMIC_VAL)->_value = NEW_VAL) +#define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ + ((ATOMIC_VAL)->_value) + +#endif /* !gcc x86 */ +#endif + +/* Standardized shortcuts. */ +#define _Py_atomic_store(ATOMIC_VAL, NEW_VAL) \ + _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_seq_cst) +#define _Py_atomic_load(ATOMIC_VAL) \ + _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_seq_cst) + +/* Python-local extensions */ + +#define _Py_atomic_store_relaxed(ATOMIC_VAL, NEW_VAL) \ + _Py_atomic_store_explicit(ATOMIC_VAL, NEW_VAL, _Py_memory_order_relaxed) +#define _Py_atomic_load_relaxed(ATOMIC_VAL) \ + _Py_atomic_load_explicit(ATOMIC_VAL, _Py_memory_order_relaxed) + +#endif /* Py_BUILD_CORE */ +#endif /* Py_ATOMIC_H */ diff --git a/python_config/pystate.h b/python_config/pystate.h new file mode 100644 index 0000000..cb1d233 --- /dev/null +++ b/python_config/pystate.h @@ -0,0 +1,279 @@ + +/* Thread and interpreter state structures and their interfaces */ + + +#ifndef Py_PYSTATE_H +#define Py_PYSTATE_H +#ifdef __cplusplus +extern "C" { +#endif + +/* State shared between threads */ + +struct _ts; /* Forward */ +struct _is; /* Forward */ + +#ifdef Py_LIMITED_API +typedef struct _is PyInterpreterState; +#else +typedef struct _is { + + struct _is *next; + struct _ts *tstate_head; + + PyObject *modules; + PyObject *modules_by_index; + PyObject *sysdict; + PyObject *builtins; + PyObject *importlib; + + PyObject *codec_search_path; + PyObject *codec_search_cache; + PyObject *codec_error_registry; + int codecs_initialized; + int fscodec_initialized; + +#ifdef HAVE_DLOPEN + int dlopenflags; +#endif +#ifdef WITH_TSC + int tscdump; +#endif + + PyObject *builtins_copy; +} PyInterpreterState; +#endif + + +/* State unique per thread */ + +struct _frame; /* Avoid including frameobject.h */ + +#ifndef Py_LIMITED_API +/* Py_tracefunc return -1 when raising an exception, or 0 for success. */ +typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *); + +/* The following values are used for 'what' for tracefunc functions: */ +#define PyTrace_CALL 0 +#define PyTrace_EXCEPTION 1 +#define PyTrace_LINE 2 +#define PyTrace_RETURN 3 +#define PyTrace_C_CALL 4 +#define PyTrace_C_EXCEPTION 5 +#define PyTrace_C_RETURN 6 +#endif + +#ifdef Py_LIMITED_API +typedef struct _ts PyThreadState; +#else +typedef struct _ts { + /* See Python/ceval.c for comments explaining most fields */ + + struct _ts *prev; + struct _ts *next; + PyInterpreterState *interp; + + struct _frame *frame; + int recursion_depth; + char overflowed; /* The stack has overflowed. Allow 50 more calls + to handle the runtime error. */ + char recursion_critical; /* The current calls must not cause + a stack overflow. */ + /* 'tracing' keeps track of the execution depth when tracing/profiling. + This is to prevent the actual trace/profile code from being recorded in + the trace/profile. */ + int tracing; + int use_tracing; + + Py_tracefunc c_profilefunc; + Py_tracefunc c_tracefunc; + PyObject *c_profileobj; + PyObject *c_traceobj; + + PyObject *curexc_type; + PyObject *curexc_value; + PyObject *curexc_traceback; + + PyObject *exc_type; + PyObject *exc_value; + PyObject *exc_traceback; + + PyObject *dict; /* Stores per-thread state */ + + int gilstate_counter; + + PyObject *async_exc; /* Asynchronous exception to raise */ + long thread_id; /* Thread id where this tstate was created */ + + int trash_delete_nesting; + PyObject *trash_delete_later; + + /* Called when a thread state is deleted normally, but not when it + * is destroyed after fork(). + * Pain: to prevent rare but fatal shutdown errors (issue 18808), + * Thread.join() must wait for the join'ed thread's tstate to be unlinked + * from the tstate chain. That happens at the end of a thread's life, + * in pystate.c. + * The obvious way doesn't quite work: create a lock which the tstate + * unlinking code releases, and have Thread.join() wait to acquire that + * lock. The problem is that we _are_ at the end of the thread's life: + * if the thread holds the last reference to the lock, decref'ing the + * lock will delete the lock, and that may trigger arbitrary Python code + * if there's a weakref, with a callback, to the lock. But by this time + * _PyThreadState_Current is already NULL, so only the simplest of C code + * can be allowed to run (in particular it must not be possible to + * release the GIL). + * So instead of holding the lock directly, the tstate holds a weakref to + * the lock: that's the value of on_delete_data below. Decref'ing a + * weakref is harmless. + * on_delete points to _threadmodule.c's static release_sentinel() function. + * After the tstate is unlinked, release_sentinel is called with the + * weakref-to-lock (on_delete_data) argument, and release_sentinel releases + * the indirectly held lock. + */ + void (*on_delete)(void *); + void *on_delete_data; + + PyObject *coroutine_wrapper; + int in_coroutine_wrapper; + + /* XXX signal handlers should also be here */ + +} PyThreadState; +#endif + + +PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); +PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *); +PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *); +PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 +/* New in 3.3 */ +PyAPI_FUNC(int) PyState_AddModule(PyObject*, struct PyModuleDef*); +PyAPI_FUNC(int) PyState_RemoveModule(struct PyModuleDef*); +#endif +PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyState_ClearModules(void); +#endif + +PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *); +PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *); +PyAPI_FUNC(void) _PyThreadState_Init(PyThreadState *); +PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *); +PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); +PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate); +#ifdef WITH_THREAD +PyAPI_FUNC(void) PyThreadState_DeleteCurrent(void); +PyAPI_FUNC(void) _PyGILState_Reinit(void); +#endif + +/* Return the current thread state. The global interpreter lock must be held. + * When the current thread state is NULL, this issues a fatal error (so that + * the caller needn't check for NULL). */ +PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); + +/* Similar to PyThreadState_Get(), but don't issue a fatal error + * if it is NULL. */ +PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); + +PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *); +PyAPI_FUNC(PyObject *) PyThreadState_GetDict(void); +PyAPI_FUNC(int) PyThreadState_SetAsyncExc(long, PyObject *); + + +/* Variable and macro for in-line access to current thread state */ + +/* Assuming the current thread holds the GIL, this is the + PyThreadState for the current thread. */ +#if defined(Py_BUILD_CORE) && !defined(__cplusplus) +PyAPI_DATA(_Py_atomic_address) _PyThreadState_Current; +# define PyThreadState_GET() \ + ((PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current)) +#else +# define PyThreadState_GET() PyThreadState_Get() +#endif + +typedef + enum {PyGILState_LOCKED, PyGILState_UNLOCKED} + PyGILState_STATE; + +#ifdef WITH_THREAD + +/* Ensure that the current thread is ready to call the Python + C API, regardless of the current state of Python, or of its + thread lock. This may be called as many times as desired + by a thread so long as each call is matched with a call to + PyGILState_Release(). In general, other thread-state APIs may + be used between _Ensure() and _Release() calls, so long as the + thread-state is restored to its previous state before the Release(). + For example, normal use of the Py_BEGIN_ALLOW_THREADS/ + Py_END_ALLOW_THREADS macros are acceptable. + + The return value is an opaque "handle" to the thread state when + PyGILState_Ensure() was called, and must be passed to + PyGILState_Release() to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles can *not* be shared - + each unique call to PyGILState_Ensure must save the handle for its + call to PyGILState_Release. + + When the function returns, the current thread will hold the GIL. + + Failure is a fatal error. +*/ +PyAPI_FUNC(PyGILState_STATE) PyGILState_Ensure(void); + +/* Release any resources previously acquired. After this call, Python's + state will be the same as it was prior to the corresponding + PyGILState_Ensure() call (but generally this state will be unknown to + the caller, hence the use of the GILState API.) + + Every call to PyGILState_Ensure must be matched by a call to + PyGILState_Release on the same thread. +*/ +PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); + +/* Helper/diagnostic function - get the current thread state for + this thread. May return NULL if no GILState API has been used + on the current thread. Note that the main thread always has such a + thread-state, even if no auto-thread-state call has been made + on the main thread. +*/ +PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); + +/* Helper/diagnostic function - return 1 if the current thread + * currently holds the GIL, 0 otherwise + */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) PyGILState_Check(void); +#endif + +#endif /* #ifdef WITH_THREAD */ + +/* The implementation of sys._current_frames() Returns a dict mapping + thread id to that thread's current frame. +*/ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void); +#endif + +/* Routines for advanced debuggers, requested by David Beazley. + Don't use unless you know what you are doing! */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Head(void); +PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Next(PyInterpreterState *); +PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *); +PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *); + +typedef struct _frame *(*PyThreadFrameGetter)(PyThreadState *self_); +#endif + +/* hook for PyEval_GetFrame(), requested for Psyco */ +#ifndef Py_LIMITED_API +PyAPI_DATA(PyThreadFrameGetter) _PyThreadState_GetFrame; +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_PYSTATE_H */ From ba7a6e5328be42692c354a6cf398c8d37dc0f0fe Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 8 Aug 2018 11:42:13 +0600 Subject: [PATCH 58/72] patchPY actually requires extractedPY --- python_build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index 7a36cdc..32e595d 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -119,10 +119,10 @@ soospatchPY: compilePY cat $(PYDIR)/Makefile_old | $(SED) 's/-Werror=declaration-after-statement//' | $(SED) 's/Python\/$$(DYNLOADFILE) \\/\\/' >$(PYDIR)/Makefile touch soospatchPY -compilePY: extractedPY patchPY +compilePY: patchPY cd $(PYDIR) && ./configure CC="$(CC)" CXX="$(CXX)" AS="$(AS)" AR="$(AR)" OBJCOPY="$(OBJCOPY)" STRIP="$(STRIP)" NM="$(NM)" RANLIB="$(RANLIB)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" ASFLAGS="$(ASFLAGS)" LDFLAGS="$(LDFLAGS)" CONFIG_SITE="config.site" --disable-shared --with-threads --without-signal-module --disable-ipv6 -host=aarch64-none-elf --build=`./config.guess` && cd .. && touch compilePY -patchPY: cloneNX +patchPY: cloneNX extractedPY cp $(PYDIR)/configure $(PYDIR)/configure_old cat $(PYDIR)/configure_old | $(SED) 's/ \*\-\*\-linux\*)/ \*\-\*\-linux\*\|aarch64\-none\-elf)/g' >$(PYDIR)/configure echo ac_cv_file__dev_ptmx=no >$(PYDIR)/config.site From a03d52d14e6264834ed397c20503c5f22f21e103 Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 8 Aug 2018 20:37:26 +0600 Subject: [PATCH 59/72] Include switch-imgui --- Makefile | 1 + python_build/Makefile | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef6044f..ed7fcc3 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,7 @@ dist: $(BUILD) mkdir -p $(LIBDIR) unzip $(PY_BUILD)/nxpy3.5.3/python.zip -d $(LIBDIR) cp -r $(PY_BUILD)/nx-*/nx $(LIBDIR) + cp -r $(PY_BUILD)/switch-imgui-*/dist/python/* $(LIBDIR) cp $(OUTPUT).nro $(DIST_DIR)/ cp $(OUTPUT).nacp $(DIST_DIR)/ cp main.py $(DIST_DIR)/ diff --git a/python_build/Makefile b/python_build/Makefile index 32e595d..5229e76 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -60,6 +60,8 @@ _NXFILE := _nx.zip _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 +IMGUIFILE := imgui.zip +IMGUIVERS := b93f16ca8f376cc13ec0a56c86940a23a99156b9 @@ -88,8 +90,10 @@ linkPY: soospatchPY cloneNX: wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NXVERS).zip" wget -O $(NXFILE) "https://github.com/nx-python/nx/archive/$(NXVERS).zip" + wget -O $(IMGUIFILE) "https://github.com/elnardu/switch-imgui/archive/$(IMGUIVERS).zip" unzip -o $(_NXFILE) unzip -o $(NXFILE) + unzip -o $(IMGUIFILE) touch cloneNX soospatchPY: compilePY @@ -130,7 +134,9 @@ patchPY: cloneNX extractedPY echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c hashtable.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\|zlib\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" | $(SED) "s/_tracemalloc/# _tracemalloc/" >$(PYDIR)/Modules/Setup.dist + cat switch-imgui-*/setup.txt >> $(PYDIR)/Modules/Setup.dist cp -r _nx-*/_nx $(PYDIR)/Modules/ + cp -r switch-imgui-*/dist/modules/* $(PYDIR)/Modules/ touch patchPY @@ -141,4 +147,4 @@ $(PYFILE): wget -O "$(PYFILE)" "$(PYLINK)" || curl -Lo "$(PYFILE)" "$(PYLINK)" clean: - @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY cloneNX _nx-* nx-* libpython*.a + @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY cloneNX _nx-* nx-* switch-imgui-* libpython*.a From d006d5f143efe06f59ab5577cb27900b8b891e9b Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 8 Aug 2018 20:49:41 +0600 Subject: [PATCH 60/72] Use updated loader --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ed7fcc3..1a16630 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ dist: $(BUILD) cp -r $(PY_BUILD)/switch-imgui-*/dist/python/* $(LIBDIR) cp $(OUTPUT).nro $(DIST_DIR)/ cp $(OUTPUT).nacp $(DIST_DIR)/ - cp main.py $(DIST_DIR)/ + cp $(PY_BUILD)/switch-imgui-*/main.py $(DIST_DIR)/ cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) #--------------------------------------------------------------------------------- From 7b0a0429734c1c2c8a3a68a4cce275b1f6684bfd Mon Sep 17 00:00:00 2001 From: elnardu Date: Wed, 8 Aug 2018 20:54:10 +0600 Subject: [PATCH 61/72] Change switch-imgui repo location --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 5229e76..80939eb 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -90,7 +90,7 @@ linkPY: soospatchPY cloneNX: wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NXVERS).zip" wget -O $(NXFILE) "https://github.com/nx-python/nx/archive/$(NXVERS).zip" - wget -O $(IMGUIFILE) "https://github.com/elnardu/switch-imgui/archive/$(IMGUIVERS).zip" + wget -O $(IMGUIFILE) "https://github.com/nx-python/switch-imgui/archive/$(IMGUIVERS).zip" unzip -o $(_NXFILE) unzip -o $(NXFILE) unzip -o $(IMGUIFILE) From f78a21902037ca1931744d2a81021903d4a4730b Mon Sep 17 00:00:00 2001 From: elnardu Date: Thu, 9 Aug 2018 23:01:54 +0600 Subject: [PATCH 62/72] Update imgui-switch version --- python_build/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_build/Makefile b/python_build/Makefile index 80939eb..0aa970d 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -61,7 +61,7 @@ _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 IMGUIFILE := imgui.zip -IMGUIVERS := b93f16ca8f376cc13ec0a56c86940a23a99156b9 +IMGUIVERS := b4e26056141604119ba272ac8fcdda712187a1a9 @@ -90,7 +90,7 @@ linkPY: soospatchPY cloneNX: wget -O $(_NXFILE) "https://github.com/nx-python/_nx/archive/$(_NXVERS).zip" wget -O $(NXFILE) "https://github.com/nx-python/nx/archive/$(NXVERS).zip" - wget -O $(IMGUIFILE) "https://github.com/nx-python/switch-imgui/archive/$(IMGUIVERS).zip" + wget -O $(IMGUIFILE) "https://github.com/nx-python/imgui-switch/archive/$(IMGUIVERS).zip" unzip -o $(_NXFILE) unzip -o $(NXFILE) unzip -o $(IMGUIFILE) From 0bf6d95ce4d8a66e3d26f5d04f7f984e237aee28 Mon Sep 17 00:00:00 2001 From: elnardu Date: Thu, 9 Aug 2018 23:05:29 +0600 Subject: [PATCH 63/72] switch-imgui -> imgui-switch --- Makefile | 4 ++-- python_build/Makefile | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 1a16630..443846b 100644 --- a/Makefile +++ b/Makefile @@ -146,10 +146,10 @@ dist: $(BUILD) mkdir -p $(LIBDIR) unzip $(PY_BUILD)/nxpy3.5.3/python.zip -d $(LIBDIR) cp -r $(PY_BUILD)/nx-*/nx $(LIBDIR) - cp -r $(PY_BUILD)/switch-imgui-*/dist/python/* $(LIBDIR) + cp -r $(PY_BUILD)/imgui-switch-*/dist/python/* $(LIBDIR) cp $(OUTPUT).nro $(DIST_DIR)/ cp $(OUTPUT).nacp $(DIST_DIR)/ - cp $(PY_BUILD)/switch-imgui-*/main.py $(DIST_DIR)/ + cp $(PY_BUILD)/imgui-switch-*/main.py $(DIST_DIR)/ cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) #--------------------------------------------------------------------------------- diff --git a/python_build/Makefile b/python_build/Makefile index 0aa970d..adb4b01 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -134,9 +134,9 @@ patchPY: cloneNX extractedPY echo ac_cv_lib_dl_dlopen=no >>$(PYDIR)/config.site cp $(PYDIR)/Modules/Setup.dist $(PYDIR)/Modules/Setup.dist_old cat $(PYDIR)/Modules/Setup.dist_old | $(SED) -e '$$a_nx -I$$(srcdir)/Modules/_nx _nx/_nxmodule.c hashtable.c' | $(SED) 's/^\([^#].* pwdmodule\.c.*\)/#\1/' | $(SED) 's/^#\(array\|cmath\|math\|_struct\|operator\|_random\|_collections\|itertools\|signal\|strop\|unicodedata\|_io\|_csv\|_md5\|_sha\|_sha256\|_sha512\|binascii\|select\|cStringIO\|time\|_functools\|_socket\|datetime\|_bisect\|zlib\)\(.*\)/\1\2/' | $(SED) "s#\\(zlib[^\$$]*\\)\$$(prefix)\\([^\$$]*\\)\$$(exec_prefix)\\(.*\\)#\1$(DEVKITPRO)/portlibs/switch\2$(DEVKITPRO)/portlibs/switch\3#" | $(SED) "s/_tracemalloc/# _tracemalloc/" >$(PYDIR)/Modules/Setup.dist - cat switch-imgui-*/setup.txt >> $(PYDIR)/Modules/Setup.dist + cat imgui-switch-*/setup.txt >> $(PYDIR)/Modules/Setup.dist cp -r _nx-*/_nx $(PYDIR)/Modules/ - cp -r switch-imgui-*/dist/modules/* $(PYDIR)/Modules/ + cp -r imgui-switch-*/dist/modules/* $(PYDIR)/Modules/ touch patchPY @@ -147,4 +147,4 @@ $(PYFILE): wget -O "$(PYFILE)" "$(PYLINK)" || curl -Lo "$(PYFILE)" "$(PYLINK)" clean: - @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY cloneNX _nx-* nx-* switch-imgui-* libpython*.a + @rm -rf $(PYDIR) $(PYFILE) $(_NXFILE) $(NXFILE) patchPY extractedPY compilePY linkPY distfPY soospatchPY cloneNX _nx-* nx-* imgui-switch-* libpython*.a From ce2bb24b80defc61edf7e3422fbdabf58c1a946d Mon Sep 17 00:00:00 2001 From: elnardu Date: Thu, 9 Aug 2018 23:27:56 +0600 Subject: [PATCH 64/72] Update imgui-switch version --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index adb4b01..1615afd 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -61,7 +61,7 @@ _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 IMGUIFILE := imgui.zip -IMGUIVERS := b4e26056141604119ba272ac8fcdda712187a1a9 +IMGUIVERS := 73a44fdb11b86744471b75f9f33f8b9273be20b9 From 3d0b398d49f64cf5122b18455e3045fddd614332 Mon Sep 17 00:00:00 2001 From: elnardu Date: Thu, 9 Aug 2018 23:42:35 +0600 Subject: [PATCH 65/72] Bump version to v0.6.0-alpha --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 443846b..9de61cc 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ INCLUDES := include EXEFS_SRC := exefs_src APP_TITLE := $(TARGET) APP_AUTHOR := nx-python, Python Software Foundation -APP_VERSION := 0.5.2-alpha +APP_VERSION := 0.6.0-alpha #--------------------------------------------------------------------------------- # options for code generation From 32187634fedb0c8b7f5f8429092fc638ac432934 Mon Sep 17 00:00:00 2001 From: elnardu Date: Sun, 12 Aug 2018 23:38:50 +0600 Subject: [PATCH 66/72] Add circleci config --- .circleci/config.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..030b0b2 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,30 @@ +version: 2 +jobs: + build: + docker: + - image: devkitpro/devkita64:20180728 + + steps: + - checkout + - run: + name: Print package versions + command: dkp-pacman -Q + + - run: + name: Install zip and unzip + command: apt-get update && apt-get install -y unzip zip + + - run: + name: Build PyNX + command: make + + - run: + name: Make dist + command: make dist + + - run: + command: cp ./build/PyNX-*.zip ./build/PyNX.zip + + - store_artifacts: + path: ./build/PyNX.zip + From 83d6d9c62e7b0b9ac7985b33394c890afc41bb3a Mon Sep 17 00:00:00 2001 From: Elnard Utiushev Date: Mon, 13 Aug 2018 00:08:46 +0600 Subject: [PATCH 67/72] Add ci badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e5ef201..d9dabfd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # PyNX +[![CircleCI branch](https://img.shields.io/circleci/project/github/nx-python/PyNX/master.svg)](https://circleci.com/gh/nx-python/PyNX) + **nx-python** is an ecosystem for developing and running Python homebrew applications on the Nintendo Switch. **PyNX** serves as the entry point to running Python apps on your Switch. It is a homebrew app that contains a port of the CPython interpreter and allows you to run Python applications from the Homebrew Menu. Just name your application `main.py` and place it next to the `PyNX.nro`, and it will be executed as soon as you launch PyNX from the Homebrew Menu. Currently, **Python 3.5 is supported**. ## Running PyNX on your Switch From a7230b367e2057ff8db3d8b205965f17360165bd Mon Sep 17 00:00:00 2001 From: elnardu Date: Thu, 16 Aug 2018 16:32:09 +0600 Subject: [PATCH 68/72] Update imgui-switch version --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 1615afd..ac9b835 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -61,7 +61,7 @@ _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 IMGUIFILE := imgui.zip -IMGUIVERS := 73a44fdb11b86744471b75f9f33f8b9273be20b9 +IMGUIVERS := 179360f82770850f1af2f03c631d01a9ff839b57 From 354284cfea4da650bd42e6daffbdf8d44d8348b9 Mon Sep 17 00:00:00 2001 From: elnardu Date: Tue, 28 Aug 2018 14:29:06 -0400 Subject: [PATCH 69/72] Move updated loader into the main repo --- Makefile | 2 +- main.py | 157 +++++++++++++++++++++++++++++++++++++------------------ 2 files changed, 108 insertions(+), 51 deletions(-) diff --git a/Makefile b/Makefile index 9de61cc..f3426e6 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ dist: $(BUILD) cp -r $(PY_BUILD)/imgui-switch-*/dist/python/* $(LIBDIR) cp $(OUTPUT).nro $(DIST_DIR)/ cp $(OUTPUT).nacp $(DIST_DIR)/ - cp $(PY_BUILD)/imgui-switch-*/main.py $(DIST_DIR)/ + cp main.py $(DIST_DIR)/ cd $(BUILD) && zip -r $(TARGET)-$(APP_VERSION).zip $(TARGET) #--------------------------------------------------------------------------------- diff --git a/main.py b/main.py index 6138e89..b849e01 100644 --- a/main.py +++ b/main.py @@ -1,67 +1,124 @@ -import sys -from glob import glob +import imgui +import imguihelper import os +import _nx import runpy +import sys import time -import _nx -import nx -from nx.utils import AnsiMenu, clear_terminal - +from imgui.integrations.nx import NXRenderer +from nx.utils import clear_terminal +import traceback sys.argv = [""] # workaround needed for runpy -TILED_DOUBLE = 1 -MAIN_PY = 'main.py' -PYNX_DIR_PATH = os.getcwd() +def colorToFloat(t): + nt = () + for v in t: + nt += ((1/255) * v, ) + return nt + +# (r, g, b) +FOLDER_COLOR = colorToFloat((230, 126, 34)) +PYFILE_COLOR = colorToFloat((46, 204, 113)) +FILE_COLOR = colorToFloat((41, 128, 185)) +ERROR = "" +TILED_DOUBLE = 1 def run_python_module(path: str): - runpy.run_path(path, run_name='__main__') + global ERROR + # clear both buffers + imguihelper.clear() + imguihelper.clear() _nx.gfx_set_mode(TILED_DOUBLE) - print("{} exited.".format(selected)) + clear_terminal() + try: + runpy.run_path(path, run_name='__main__') + except Exception as e: + ERROR = traceback.format_exc() + imguihelper.initialize() -def confirmation_prompt(program_name: str): - name = program_name[:-3] if program_name.endswith('.py') else program_name - name = name[:-1] if name.endswith(os.sep) else name - print("Do you want to execute the Python program {}?\n" - "Press A to confirm, B to cancel.".format(name)) +def main(): + global ERROR + renderer = NXRenderer() + currentDir = os.getcwd() + while True: - nx.refresh_inputs() - if nx.p1.a_button.is_pressed: - return True - elif nx.p1.b_button.is_pressed: - return False - time.sleep(0.01) + renderer.handleinputs() + imgui.new_frame() -if __name__ == '__main__': - os.chdir(os.pardir) + width, height = renderer.io.display_size + imgui.set_next_window_size(width, height) + imgui.set_next_window_position(0, 0) + imgui.begin("", + flags=imgui.WINDOW_NO_TITLE_BAR | imgui.WINDOW_NO_RESIZE | imgui.WINDOW_NO_MOVE | imgui.WINDOW_NO_SAVED_SETTINGS + ) + imgui.text("Welcome to PyNX!") + imgui.text("Touch is supported") + imgui.text("Current dir: " + os.getcwd()) - while True: - clear_terminal() - dirs = [os.pardir + '/'] + glob('*/') - scripts = glob('*.py') - if os.getcwd() == PYNX_DIR_PATH and MAIN_PY in scripts: - scripts.remove(MAIN_PY) - for _dir in dirs: - if os.path.isfile(os.path.join(_dir, MAIN_PY)): - dirs[dirs.index(_dir)] = _dir[:-1] - listing = dirs + scripts - file_menu = AnsiMenu(listing) - print("Listing for {}:".format(os.getcwd())) - selected = listing[file_menu.query()] + if os.getcwd() != "sdmc:/": + imgui.push_style_color(imgui.COLOR_BUTTON, *FOLDER_COLOR) + if imgui.button("../", width=200, height=60): + os.chdir("..") + imgui.pop_style_color(1) + + dirs = [] + files = [] + for e in os.listdir(): + if os.path.isdir(e): + dirs.append(e) + else: + files.append(e) + + dirs = sorted(dirs) + files = sorted(files) + + for e in dirs: + imgui.push_style_color(imgui.COLOR_BUTTON, *FOLDER_COLOR) + if imgui.button(e + "/", width=200, height=60): + os.chdir(e) + imgui.pop_style_color(1) + + for e in files: + if e.endswith(".py"): + imgui.push_style_color(imgui.COLOR_BUTTON, *PYFILE_COLOR) + else: + imgui.push_style_color(imgui.COLOR_BUTTON, *FILE_COLOR) + + if imgui.button(e, width=200, height=60) and e.endswith(".py"): + run_python_module(e) + + imgui.pop_style_color(1) + + + imgui.end() + + if ERROR: + imgui.set_next_window_size(width, height) + imgui.set_next_window_position(0, 0) + imgui.begin("ERROR", + flags=imgui.WINDOW_NO_RESIZE | imgui.WINDOW_NO_MOVE | imgui.WINDOW_NO_SAVED_SETTINGS + ) + imgui.text(str(ERROR)) + if imgui.button("OK", width=200, height=60): + ERROR = "" + imgui.end() + + imgui.render() + renderer.render() + + renderer.shutdown() + + +if __name__ == "__main__": + try: + main() + except Exception: + imguihelper.clear() + imguihelper.clear() + _nx.gfx_set_mode(TILED_DOUBLE) clear_terminal() - if selected in dirs: - os.chdir(selected) - if os.path.isfile(MAIN_PY) and not selected == os.pardir + '/' and not os.getcwd() == PYNX_DIR_PATH: - response = confirmation_prompt(selected) - clear_terminal() - if response is True: - run_python_module(MAIN_PY) - os.chdir(os.pardir) - else: - response = confirmation_prompt(selected) - clear_terminal() - if response is True: - run_python_module(selected) + traceback.print_exc() From c8fe948170deab1c919d1247c6beb91918398ce7 Mon Sep 17 00:00:00 2001 From: elnardu Date: Tue, 28 Aug 2018 22:56:03 -0400 Subject: [PATCH 70/72] Add "Interactive python" button --- main.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b849e01..8d34b5d 100644 --- a/main.py +++ b/main.py @@ -6,7 +6,7 @@ import sys import time from imgui.integrations.nx import NXRenderer -from nx.utils import clear_terminal +from nx.utils import clear_terminal, Terminal import traceback sys.argv = [""] # workaround needed for runpy @@ -21,6 +21,7 @@ def colorToFloat(t): FOLDER_COLOR = colorToFloat((230, 126, 34)) PYFILE_COLOR = colorToFloat((46, 204, 113)) FILE_COLOR = colorToFloat((41, 128, 185)) +APP_COLOR = colorToFloat((24, 86, 31)) ERROR = "" @@ -55,6 +56,8 @@ def main(): imgui.begin("", flags=imgui.WINDOW_NO_TITLE_BAR | imgui.WINDOW_NO_RESIZE | imgui.WINDOW_NO_MOVE | imgui.WINDOW_NO_SAVED_SETTINGS ) + imgui.begin_group() + imgui.text("Welcome to PyNX!") imgui.text("Touch is supported") imgui.text("Current dir: " + os.getcwd()) @@ -92,6 +95,21 @@ def main(): run_python_module(e) imgui.pop_style_color(1) + + imgui.end_group() + # end of file picker + + imgui.same_line(spacing=50) + imgui.begin_group() + imgui.text("Utils:") + + imgui.push_style_color(imgui.COLOR_BUTTON, *APP_COLOR) + if imgui.button("Interactive Python", width=200, height=60): + t = Terminal() + t.main() + imgui.pop_style_color(1) + + imgui.end_group() imgui.end() From 93c8e6d128f347f84371c05091ad3023b2f44b2a Mon Sep 17 00:00:00 2001 From: elnardu Date: Tue, 28 Aug 2018 23:00:32 -0400 Subject: [PATCH 71/72] Update nx package version --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index ac9b835..09b7227 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -59,7 +59,7 @@ PYFILE := py.tgz _NXFILE := _nx.zip _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip -NXVERS := e0fb67e90e945c6f214f16a804ae1ca7fa211d95 +NXVERS := 8ecd6ea46e620cfad4732a9dd7d1ee228309cb1b IMGUIFILE := imgui.zip IMGUIVERS := 179360f82770850f1af2f03c631d01a9ff839b57 From b2a76325b1e395c59aff6ea10cc506c286f5a72f Mon Sep 17 00:00:00 2001 From: Elnard Utiushev Date: Sun, 2 Sep 2018 17:00:59 -0400 Subject: [PATCH 72/72] Update nx package --- python_build/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_build/Makefile b/python_build/Makefile index 09b7227..74bee60 100644 --- a/python_build/Makefile +++ b/python_build/Makefile @@ -59,7 +59,7 @@ PYFILE := py.tgz _NXFILE := _nx.zip _NXVERS := 5baa28b0ce6ab2c8390fd85cfc36e1086f2cede9 NXFILE := nx.zip -NXVERS := 8ecd6ea46e620cfad4732a9dd7d1ee228309cb1b +NXVERS := eef890370950b2dec8be797d1e26113cfe457c2f IMGUIFILE := imgui.zip IMGUIVERS := 179360f82770850f1af2f03c631d01a9ff839b57