Profile #notests

memray and dependencies

trace sync_service

trace_python_allocators

Disable tracing

Control memray with SIGUSR1

Dump threads

Start and stop tracker using signal

Use native thread ids, dump threads through HTTP

trace from the start

disable autotracing

memray and dependencies

trace sync_service

trace_python_allocators

Disable tracing

Control memray with SIGUSR1

Dump threads

Start and stop tracker using signal

Use native thread ids, dump threads through HTTP

trace from the start

disable autotracing

pyinstrument #notests

Fix profiling #notests

py-spy #notests

malloc_stats #notests

threading.stack_size #notests

sys._debugmallocstats() #notests

track from main #notests

Don't track from start #notests

Malloc trim #notests

malloc_trim.py

Remove trim from malloc_stats #notests

Profile

bin/inbox-start.py

@@ -6,10 +6,18 @@
 import signal
 import socket
 import sys
+import threading
+import time
 
 import click
+import memray
 import setproctitle
 
+from inbox.glibc_malloc import (
+    maybe_start_malloc_stats_thread,
+    maybe_start_malloc_trim_thread,
+)
+
 # Check that the inbox package is installed. It seems Vagrant may sometimes
 # fail to provision the box appropriately; this check is a reasonable
 # approximation of "Did the setup script run?"
@@ -22,6 +30,7 @@
         "Try running sudo ./setup.sh"
     )
 
+import inbox.thread_inspector
 from inbox.error_handling import maybe_enable_rollbar
 from inbox.logging import configure_logging, get_logger
 from inbox.mailsync.frontend import SyncHTTPFrontend
@@ -72,6 +81,8 @@
 )
 def main(prod, enable_profiler, config, process_num):
     """Launch the Nylas sync service."""
+    threading.stack_size(524288)
+
     level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL"))
     configure_logging(log_level=level)
     reconfigure_logging()
@@ -117,6 +128,11 @@ def main(prod, enable_profiler, config, process_num):
 
     signal.signal(signal.SIGTERM, lambda *_: sync_service.stop())
     signal.signal(signal.SIGINT, lambda *_: sync_service.stop())
+    signal.signal(signal.SIGUSR1, lambda *_: track_memory())
+    signal.signal(signal.SIGUSR2, lambda *_: dump_threads())
+
+    maybe_start_malloc_stats_thread()
+    maybe_start_malloc_trim_thread()
 
     http_frontend = SyncHTTPFrontend(sync_service, port, enable_profiler_api)
     http_frontend.start()
@@ -126,5 +142,26 @@ def main(prod, enable_profiler, config, process_num):
     print("\033[94mNylas Sync Engine exiting...\033[0m", file=sys.stderr)
 
 
+tracker = None
+
+
+def track_memory():
+    global tracker
+
+    if not tracker:
+        tracker = memray.Tracker(
+            f"bin/inbox-start-{int(time.time())}.bin", trace_python_allocators=True
+        )
+        tracker.__enter__()
+    else:
+        tracker.__exit__(None, None, None)
+        tracker = None
+
+
+def dump_threads():
+    for thread in inbox.thread_inspector.enumerate():
+        print("-->", thread, hex(thread.native_id))
+
+
 if __name__ == "__main__":
     main()

inbox/glibc_malloc.py

@@ -0,0 +1,127 @@
+import ctypes
+import sys
+import threading
+import time
+from functools import cache
+from typing import NoReturn
+
+from inbox.logging import get_logger
+
+logger = get_logger()
+
+
+class MallInfo(ctypes.Structure):
+    _fields_ = [
+        (name, ctypes.c_int)
+        for name in (
+            "arena",
+            "ordblks",
+            "smblks",
+            "hblks",
+            "hblkhd",
+            "usmblks",
+            "fsmblks",
+            "uordblks",
+            "fordblks",
+            "keepcost",
+        )
+    ]
+
+
+@cache
+def get_glibc() -> "ctypes.CDLL | None":
+    try:
+        libc = ctypes.CDLL("libc.so.6")
+    except OSError:
+        logger.warning("Not Linux, not starting malloc_trim thread")
+        return None
+
+    try:
+        libc.gnu_get_libc_version
+    except AttributeError:
+        logger.warning(
+            "Not glibc or glibc version too old, not starting malloc_trim thread"
+        )
+        return None
+
+    # https://man7.org/linux/man-pages/man3/gnu_get_libc_version.3.html
+    # since glibc 2.1
+    gnu_get_libc_version = libc.gnu_get_libc_version
+    gnu_get_libc_version.argtypes = []
+    gnu_get_libc_version.restype = ctypes.c_char_p
+
+    logger.info("glibc", version=gnu_get_libc_version().decode())
+
+    # https://man7.org/linux/man-pages/man3/mallinfo.3.html
+    # since glibc 2.0
+    mallinfo = libc.mallinfo
+    mallinfo.argtypes = []
+    mallinfo.restype = MallInfo
+
+    # https://man7.org/linux/man-pages/man3/malloc_stats.3.html
+    # since glibc 2.0
+    libc.malloc_stats.restype = None
+
+    # https://man7.org/linux/man-pages/man3/malloc_trim.3.html
+    # since glibc 2.0
+    malloc_trim = libc.malloc_trim
+    malloc_trim.argtypes = [ctypes.c_size_t]
+    malloc_trim.restype = ctypes.c_int
+
+    return libc
+
+
+MEGABYTE = 1024 * 1024
+
+
+def periodically_run_malloc_trim(libc: ctypes.CDLL, pad: int) -> NoReturn:
+    while True:
+        time.sleep(120)
+
+        mallinfo_result = libc.mallinfo()
+        if mallinfo_result.keepcost > pad:
+            libc.malloc_trim(pad)
+
+
+def maybe_start_malloc_trim_thread() -> "threading.Thread | None":
+    libc = get_glibc()
+    if libc is None:
+        return None
+
+    malloc_trim_thread = threading.Thread(
+        target=periodically_run_malloc_trim,
+        args=(libc, MEGABYTE),
+        name="malloc_trim",
+        daemon=True,
+    )
+    malloc_trim_thread.start()
+
+    logger.info("Started malloc trim_thread")
+
+    return malloc_trim_thread
+
+
+def maybe_start_malloc_stats_thread() -> "threading.Thread | None":
+    libc = get_glibc()
+    if libc is None:
+        return None
+
+    malloc_stats_thread = threading.Thread(
+        target=periodicallty_print_malloc_stats, args=(libc,), daemon=True
+    )
+    malloc_stats_thread.start()
+
+    return malloc_stats_thread
+
+
+def periodicallty_print_malloc_stats(libc):
+    while True:
+        libc.malloc_stats()
+        info = libc.mallinfo()
+        fields = [(name, getattr(info, name)) for name, _ in info._fields_]
+        print("Malloc info:")
+        for name, value in fields:
+            print(f"- {name}: {value}")
+        print("sys._debugmallocstats()")
+        sys._debugmallocstats()
+        time.sleep(60)

inbox/mailsync/frontend.py

@@ -4,6 +4,7 @@
 from pympler import muppy, summary
 from werkzeug.serving import WSGIRequestHandler, run_simple
 
+import inbox.thread_inspector
 from inbox.instrumentation import ProfileCollector
 
 
@@ -53,6 +54,16 @@ def mem():
             summ = summary.summarize(objs)
             return "\n".join(summary.format_(summ)) + "\n"
 
+        @app.route("/dump-threads")
+        def dump_threads():
+            return (
+                "\n".join(
+                    f"{t!r}, {hex(t.native_id)}"
+                    for t in inbox.thread_inspector.enumerate()
+                )
+                + "\n"
+            )
+
 
 class SyncbackHTTPFrontend(ProfilingHTTPFrontend):
     pass

inbox/thread_inspector.py

@@ -0,0 +1,9 @@
+import threading
+
+
+def enumerate():
+    for thread in threading.enumerate():
+        if thread.daemon or thread is threading.main_thread():
+            continue
+
+        yield thread

requirements/prod.txt

@@ -47,6 +47,12 @@ Mako==1.2.2
 MarkupSafe==2.1.2
 matplotlib-inline==0.1.6
 mysqlclient==2.2.5
+memray==1.14.0
+rich==13.9.2
+textual==0.82.0
+markdown-it-py==3.0.0
+mdurl==0.1.2
+platformdirs==4.3.6
 parso==0.8.3
 pexpect==4.8.0
 pickleshare==0.7.5
@@ -59,6 +65,7 @@ pycparser==2.21
 pygments==2.17.2
 Pympler==0.9
 PyNaCl==1.5.0
+py-spy==0.3.14
 python-dateutil==2.8.2
 pytz==2021.3
 PyYAML==6.0.2
@@ -77,7 +84,7 @@ stack-data==0.6.2
 tldextract==3.1.2
 structlog==21.4.0
 traitlets==5.9.0
-typing_extensions==4.0.1
+typing_extensions==4.12.2
 urllib3==1.26.20
 vobject==0.9.6.1
 wcwidth==0.2.6