First commit

Author: levitation

.gitignore

@@ -0,0 +1 @@
+/__pycache__

linux_rlimit.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+#
+# Roland Pihlakas, 2018
+# roland@simplify.ee
+#
+
+
+# Set memory limits for current process,  to specified 'size' in bytes
+def set_mem_limits(data_size, address_space_size):
+
+
+  import resource
+
+
+  rlimits = [
+    (resource.RLIMIT_DATA, (int(data_size), int(data_size))) if data_size else None,
+    (resource.RLIMIT_AS, (int(address_space_size), int(address_space_size))) if address_space_size else None,   # The maximum area (in bytes) of address space which may be taken by the process.   # includes memory that is from shared libraries - https://stackoverflow.com/questions/7880784/what-is-rss-and-vsz-in-linux-memory-management
+
+    # (resource.RLIMIT_RSS, (int(mem_limit), int(mem_limit))),   # The maximum resident set size that should be made available to the process.  # Ignored in Linux 2.4.30 and higher - https://superuser.com/questions/239796/limits-conf-to-set-memory-limits
+
+    # (resource.RLIMIT_STACK, (int(8 * 1024 * 1024), int(8 * 1024 * 1024))),
+    # (resource.RLIMIT_MEMLOCK, (int(64 * 1024), int(64 * 1024))),   # The maximum address space which may be locked in memory.  # 64kB is the default soft and hard limit
+    # (resource.RLIMIT_MSGQUEUE, (0, 0)),   # The number of bytes that can be allocated for POSIX message queues.)
+    # (resource.RLIMIT_SIGPENDING, (0, 0)),
+    # (resource.RLIMIT_NICE, (10, 10)),   # -20 is the highest priority and 19 is the lowest priority
+    # (resource.RLIMIT_SBSIZE, (0, 0)),   # The maximum size (in bytes) of socket buffer usage for this user. This limits the amount of network memory, and hence the amount of mbufs, that this user may hold at any time.  # Availability: FreeBSD 9 or later.
+    # (resource.RLIMIT_CPU, (cpu_timelimit, cpu_timelimit)),
+    # (resource.RLIMIT_NPROC, (0, 0)),  # disable subprocess creation
+
+    # (resource.RLIMIT_FSIZE, (0, 0)),   # disable file writing
+    # (resource.RLIMIT_NOFILE, (16, 16)),   # The maximum number of open file descriptors for the current process.   # 1024 is the default soft limit, so lets make it a hard limit
+    # (resource.RLIMIT_CORE, (0, 0)),  # The maximum size (in bytes) of a core file that the current process can create. 
+  ]
+
+
+  for rlimit in rlimits:
+    if rlimit:  # not None
+      try:        
+        resource.setrlimit(rlimit[0], rlimit[1])
+      except Exception as msg:
+        print(msg)
+        pass
+
+
+#/ def set_mem_limits(data_size, address_space_size):

main.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+
+
+
+if __name__ == "__main__":  # subprocesses are automatically included in the aggregated accounting of the limits set to the main process
+
+  if (1 == 1):   # prevent system hangs
+  
+    try:
+
+      import os
+
+      if os.name == 'nt':
+
+        from windows_jobobject import set_mem_commit_limit
+
+        mem_limit = 4 * 1024 * 1024 * 1024
+        min_free_swap = 8 * 1024 * 1024 * 1024
+
+        set_mem_commit_limit(os.getpid(), mem_limit, min_free_swap)
+
+      else:   #/ if os.name == 'nt':
+
+        from linux_rlimit import set_mem_limits
+
+        data_size_limit = 512 * 1024 * 1024
+        address_space_size_limit = 1024 * 1024 * 1024
+
+        set_mem_limits(data_size_limit, address_space_size_limit)
+
+      #/ if os.name == 'nt':
+
+    except Exception as msg:
+      print(msg)
+      pass
+
+  #/ if (1 == 1):
+
+#/ if __name__ == "__main__":
+
+
+
+
+
+# unleash your dragons here
+
+print("hello")
+
+
+
+

windows_jobobject.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+#
+# Roland Pihlakas, 2017
+# roland@simplify.ee
+#
+
+
+# https://stackoverflow.com/questions/16779497/how-to-set-memory-limit-for-thread-or-process-in-python
+
+import ctypes
+import os
+import psutil
+
+
+
+PROCESS_SET_QUOTA = 0x100
+PROCESS_TERMINATE = 0x1
+JobObjectExtendedLimitInformation = 9
+
+
+# https://msdn.microsoft.com/en-us/library/windows/desktop/ms684147(v=vs.85).aspx
+
+JOB_OBJECT_LIMIT_PROCESS_MEMORY = 0x00000100
+JOB_OBJECT_LIMIT_JOB_MEMORY = 0x00000200
+
+JOB_OBJECT_LIMIT_WORKINGSET = 0x00000001
+
+JOB_OBJECT_LIMIT_PRESERVE_JOB_TIME = 0x00000040   # Preserves any job time limits you previously set. As long as this flag is set, you can establish a per-job time limit once, then alter other limits in subsequent calls. This flag cannot be used with JOB_OBJECT_LIMIT_JOB_TIME.
+JOB_OBJECT_LIMIT_JOB_TIME = 0x00000004
+JOB_OBJECT_LIMIT_PROCESS_TIME = 0x00000002
+
+JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 
+
+
+# https://stackoverflow.com/questions/6266820/working-example-of-createjobobject-setinformationjobobject-pinvoke-in-net
+JOB_OBJECT_LIMIT_ACTIVE_PROCESS = 0x00000008 
+JOB_OBJECT_LIMIT_AFFINITY = 0x00000010 
+JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800 
+JOB_OBJECT_LIMIT_DIE_ON_UNHANDLED_EXCEPTION = 0x00000400 
+JOB_OBJECT_LIMIT_JOB_MEMORY = 0x00000200 
+JOB_OBJECT_LIMIT_JOB_TIME = 0x00000004 
+JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 
+JOB_OBJECT_LIMIT_PRESERVE_JOB_TIME = 0x00000040 
+JOB_OBJECT_LIMIT_PRIORITY_CLASS = 0x00000020 
+JOB_OBJECT_LIMIT_PROCESS_MEMORY = 0x00000100 
+JOB_OBJECT_LIMIT_PROCESS_TIME = 0x00000002 
+JOB_OBJECT_LIMIT_SCHEDULING_CLASS = 0x00000080 
+JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x00001000 
+JOB_OBJECT_LIMIT_WORKINGSET = 0x00000001
+
+
+class IO_COUNTERS(ctypes.Structure):
+  _fields_ = [('ReadOperationCount', ctypes.c_uint64),
+              ('WriteOperationCount', ctypes.c_uint64),
+              ('OtherOperationCount', ctypes.c_uint64),
+              ('ReadTransferCount', ctypes.c_uint64),
+              ('WriteTransferCount', ctypes.c_uint64),
+              ('OtherTransferCount', ctypes.c_uint64)]
+
+
+class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
+  _fields_ = [('PerProcessUserTimeLimit', ctypes.c_int64),
+              ('PerJobUserTimeLimit', ctypes.c_int64),
+              ('LimitFlags', ctypes.c_uint32),
+              ('MinimumWorkingSetSize', ctypes.c_void_p),
+              ('MaximumWorkingSetSize', ctypes.c_void_p),
+              ('ActiveProcessLimit', ctypes.c_uint32),
+              ('Affinity', ctypes.c_void_p),
+              ('PriorityClass', ctypes.c_uint32),
+              ('SchedulingClass', ctypes.c_uint32)]
+
+
+class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
+  _fields_ = [('BasicLimitInformation', JOBOBJECT_BASIC_LIMIT_INFORMATION),
+              ('IoInfo', IO_COUNTERS),
+              ('ProcessMemoryLimit', ctypes.c_void_p),
+              ('JobMemoryLimit', ctypes.c_void_p),
+              ('PeakProcessMemoryUsed', ctypes.c_void_p),
+              ('PeakJobMemoryUsed', ctypes.c_void_p)]
+
+## https://msdn.microsoft.com/en-us/library/windows/desktop/hh448384(v=vs.85).aspx
+#class JOBOBJECT_CPU_RATE_CONTROL_INFORMATION(ctypes.Structure):
+#  _fields_ = [('BasicLimitInformation', JOBOBJECT_BASIC_LIMIT_INFORMATION),
+#              ('IoInfo', IO_COUNTERS),
+#              ('ProcessMemoryLimit', ctypes.c_void_p),
+#              ('JobMemoryLimit', ctypes.c_void_p),
+#              ('PeakProcessMemoryUsed', ctypes.c_void_p),
+#              ('PeakJobMemoryUsed', ctypes.c_void_p)]
+
+
+
+# Set memory limit for process with specfied 'pid', to specified 'size' in bytes
+def set_mem_commit_limit(pid, size, min_free_swap = None, retry_count = 10):
+
+  for i in range(0, retry_count):   # assigning to job fails sometimes
+    try:
+      if (set_mem_commit_limit_worker(pid, size, min_free_swap)):
+        break
+    except Exception as msg:  # possibly already assigned to job object? (this can be done only once)
+      print(msg)
+      pass
+
+#/ def set_mem_commit_limit(pid, size, min_free_swap, retry_count):
+
+
+def set_mem_commit_limit_worker(pid, size, min_free_swap = None):
+
+
+  if min_free_swap:
+    swap_status = psutil.swap_memory()
+    max_commit_limit = swap_status.free - min_free_swap
+    size = min(size, max_commit_limit)  # NB!
+
+
+  job_info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
+  out_size = ctypes.c_uint32()
+
+
+  # child processes inherit the job object so it is not necessary to check for the parent
+  main_pid = os.getpid() # if __name__ == '__main__' else os.getppid()
+
+
+  # https://msdn.microsoft.com/en-us/library/windows/desktop/ms682409(v=vs.85).aspx
+  # If the object existed before the function call, the function returns a handle to the existing job object and GetLastError returns ERROR_ALREADY_EXISTS.
+  job = ctypes.windll.kernel32.CreateJobObjectW(None, None)   # NB! CreateJobObjectW not CreateJobObjectA
+  # assert job != 0
+
+  success = ctypes.windll.kernel32.QueryInformationJobObject(
+    job,
+    JobObjectExtendedLimitInformation,
+    ctypes.POINTER(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)(job_info),
+    ctypes.sizeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION),
+    ctypes.POINTER(ctypes.c_uint32)(out_size)
+  )
+  # assert success
+
+
+  #job_info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PROCESS_MEMORY | JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+  #job_info.ProcessMemoryLimit  = size
+
+  job_info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_JOB_MEMORY | JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+  job_info.JobMemoryLimit = size
+
+  # job_info.MaximumWorkingSetSize = size
+
+
+  success = ctypes.windll.kernel32.SetInformationJobObject(
+    job,
+    JobObjectExtendedLimitInformation,
+    ctypes.POINTER(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)(job_info),
+    ctypes.sizeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)
+  )
+  # assert success
+
+  process = ctypes.windll.kernel32.OpenProcess(
+    PROCESS_SET_QUOTA | PROCESS_TERMINATE,
+    False, pid
+  )
+  # assert process != 0
+
+  success_assign = ctypes.windll.kernel32.AssignProcessToJobObject(job, process)
+  # assert success
+
+  #if __name__ != '__main__': # NB! due to JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE flag
+  # success = ctypes.windll.kernel32.CloseHandle(job)
+  # # assert success
+
+  success = ctypes.windll.kernel32.CloseHandle(process)
+  # assert success
+
+  return success_assign
+
+#/ def set_mem_commit_limit(pid, size, min_free_swap):