From fe1158fb6747a2804b34278e0a48ad9754243e06 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 1 Nov 2024 23:10:31 +0100 Subject: [PATCH 01/64] Decouple workers. --- frankenphp.c | 84 ++++++++++++++++++++++++++++----------------- frankenphp.go | 36 ++++++++++++++++++-- frankenphp.h | 4 ++- php_thread.go | 15 +++++++- worker.go | 94 +++++++++++++++++++++++++++++++++++++++++++-------- 5 files changed, 183 insertions(+), 50 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 54c149763..ce71f7bec 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -838,10 +838,10 @@ static void set_thread_name(char *thread_name) { #endif } -static void *php_thread(void *arg) { - char thread_name[16] = {0}; - snprintf(thread_name, 16, "php-%" PRIxPTR, (uintptr_t)arg); +static void init_php_thread(void *arg) { thread_index = (uintptr_t)arg; + char thread_name[16] = {0}; + snprintf(thread_name, 16, "php-%" PRIxPTR, thread_index); set_thread_name(thread_name); #ifdef ZTS @@ -853,14 +853,41 @@ static void *php_thread(void *arg) { #endif local_ctx = malloc(sizeof(frankenphp_server_context)); +} +static void shutdown_php_thread(void) { + //free(local_ctx); + //local_ctx = NULL; +#ifdef ZTS + ts_free_thread(); +#endif +} +static void *php_thread(void *arg) { + init_php_thread(arg); + + // handle requests until the channel is closed while (go_handle_request(thread_index)) { } -#ifdef ZTS - ts_free_thread(); -#endif + shutdown_php_thread(); + return NULL; +} + +static void *php_worker_thread(void *arg) { + init_php_thread(arg); + + // run the loop that executes the worker script + while (true) { + char *script_name = go_before_worker_script(thread_index); + if (script_name == NULL) { + break; + } + frankenphp_execute_script(script_name); + go_after_worker_script(thread_index); + } + shutdown_php_thread(); + go_shutdown_woker_thread(thread_index); return NULL; } @@ -912,28 +939,7 @@ static void *php_main(void *arg) { frankenphp_sapi_module.startup(&frankenphp_sapi_module); - pthread_t *threads = malloc(num_threads * sizeof(pthread_t)); - if (threads == NULL) { - perror("malloc failed"); - exit(EXIT_FAILURE); - } - - for (uintptr_t i = 0; i < num_threads; i++) { - if (pthread_create(&(*(threads + i)), NULL, &php_thread, (void *)i) != 0) { - perror("failed to create PHP thread"); - free(threads); - exit(EXIT_FAILURE); - } - } - - for (int i = 0; i < num_threads; i++) { - if (pthread_join((*(threads + i)), NULL) != 0) { - perror("failed to join PHP thread"); - free(threads); - exit(EXIT_FAILURE); - } - } - free(threads); + go_listen_for_shutdown(); /* channel closed, shutdown gracefully */ frankenphp_sapi_module.shutdown(&frankenphp_sapi_module); @@ -955,19 +961,35 @@ static void *php_main(void *arg) { return NULL; } -int frankenphp_init(int num_threads) { +int frankenphp_new_main_thread(int num_threads) { pthread_t thread; if (pthread_create(&thread, NULL, &php_main, (void *)(intptr_t)num_threads) != 0) { go_shutdown(); - return -1; } - return pthread_detach(thread); } +int frankenphp_new_worker_thread(uintptr_t thread_index){ + pthread_t thread; + if (pthread_create(&thread, NULL, &php_worker_thread, (void *)thread_index) != 0){ + return 1; + } + pthread_detach(thread); + return 0; +} + +int frankenphp_new_php_thread(uintptr_t thread_index){ + pthread_t thread; + if (pthread_create(&thread, NULL, &php_thread, (void *)thread_index) != 0){ + return 1; + } + pthread_detach(thread); + return 0; +} + int frankenphp_request_startup() { if (php_request_startup() == SUCCESS) { return SUCCESS; diff --git a/frankenphp.go b/frankenphp.go index 2882a1c17..5bbbaeae0 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -65,6 +65,7 @@ var ( requestChan chan *http.Request done chan struct{} + mainThreadWG sync.WaitGroup shutdownWG sync.WaitGroup loggerMu sync.RWMutex @@ -336,8 +337,13 @@ func Init(options ...Option) error { requestChan = make(chan *http.Request) initPHPThreads(opt.numThreads) - if C.frankenphp_init(C.int(opt.numThreads)) != 0 { - return MainThreadCreationError + startMainThread(opt.numThreads) + + // TODO: calc num threads + for i := 0; i < 1; i++ { + if err := startNewThread(); err != nil { + return err + } } if err := initWorkers(opt.workers); err != nil { @@ -386,6 +392,24 @@ func drainThreads() { phpThreads = nil } +func startMainThread(numThreads int) error { + mainThreadWG.Add(1) + if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { + return MainThreadCreationError + } + mainThreadWG.Wait() + return nil +} + +func startNewThread() error { + thread := getInactiveThread() + thread.isActive = true + if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("error creating thread %d", thread.threadIndex) + } + return nil +} + func getLogger() *zap.Logger { loggerMu.RLock() defer loggerMu.RUnlock() @@ -505,6 +529,14 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error return nil } +//export go_listen_for_shutdown +func go_listen_for_shutdown(){ + mainThreadWG.Done() + select{ + case <-done: + } +} + //export go_putenv func go_putenv(str *C.char, length C.int) C.bool { // Create a byte slice from C string with a specified length diff --git a/frankenphp.h b/frankenphp.h index a0c54936d..7470ba00e 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -40,7 +40,9 @@ typedef struct frankenphp_config { } frankenphp_config; frankenphp_config frankenphp_get_config(); -int frankenphp_init(int num_threads); +int frankenphp_new_main_thread(int num_threads); +int frankenphp_new_php_thread(uintptr_t thread_index); +int frankenphp_new_worker_thread(uintptr_t thread_index); int frankenphp_update_server_context( bool create, bool has_main_request, bool has_active_request, diff --git a/php_thread.go b/php_thread.go index 5b9c29970..608040b93 100644 --- a/php_thread.go +++ b/php_thread.go @@ -15,15 +15,28 @@ type phpThread struct { mainRequest *http.Request workerRequest *http.Request worker *worker + isActive bool + isReady bool + threadIndex int } func initPHPThreads(numThreads int) { phpThreads = make([]*phpThread, 0, numThreads) for i := 0; i < numThreads; i++ { - phpThreads = append(phpThreads, &phpThread{}) + phpThreads = append(phpThreads, &phpThread{threadIndex: i}) } } +func getInactiveThread() *phpThread { + for _, thread := range phpThreads { + if !thread.isActive { + return thread + } + } + + return nil +} + func (thread phpThread) getActiveRequest() *http.Request { if thread.workerRequest != nil { return thread.workerRequest diff --git a/worker.go b/worker.go index 38e4b60a4..0dd71d1fb 100644 --- a/worker.go +++ b/worker.go @@ -47,9 +47,10 @@ func initWorkers(opt []workerOpt) error { if err != nil { return err } - workersReadyWG.Add(worker.num) for i := 0; i < worker.num; i++ { - go worker.startNewWorkerThread() + if err := worker.startNewThread(nil); err != nil { + return err + } } } @@ -82,6 +83,19 @@ func newWorker(o workerOpt) (*worker, error) { return w, nil } +func (worker *worker) startNewThread(r *http.Request) error { + workersReadyWG.Add(1) + workerShutdownWG.Add(1) + thread := getInactiveThread() + thread.worker = worker + thread.isActive = true + if C.frankenphp_new_worker_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("failed to create worker thread") + } + + return nil +} + func (worker *worker) startNewWorkerThread() { workerShutdownWG.Add(1) defer workerShutdownWG.Done() @@ -232,26 +246,76 @@ func restartWorkers(workerOpts []workerOpt) { } func assignThreadToWorker(thread *phpThread) { - fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) - metrics.ReadyWorker(fc.scriptFilename) - worker, ok := workers[fc.scriptFilename] - if !ok { - panic("worker not found for script: " + fc.scriptFilename) - } - thread.worker = worker - if !workersAreReady.Load() { - workersReadyWG.Done() - } + metrics.ReadyWorker(thread.worker.fileName) + thread.isReady = true + workersReadyWG.Done() // TODO: we can also store all threads assigned to the worker if needed } +//export go_before_worker_script +func go_before_worker_script(threadIndex C.uintptr_t) *C.char { + thread := phpThreads[threadIndex] + worker := thread.worker + + // if we are done, exit the loop that restarts the worker script + if workersAreDone.Load() { + return nil + } + metrics.StartWorker(worker.fileName) + + // Create main dummy request + r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) + if err != nil { + panic(err) + } + + r, err = NewRequestWithContext( + r, + WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), + WithRequestPreparedEnv(worker.env), + ) + if err != nil { + panic(err) + } + thread.mainRequest = r + if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { + c.Write(zap.String("worker", worker.fileName), zap.Int("num", worker.num)) + } + + if err := updateServerContext(r, true, false); err != nil { + panic(err) + } + return C.CString(worker.fileName) +} + +//export go_after_worker_script +func go_after_worker_script(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + + // on exit status 0 we just run the worker script again + if fc.exitStatus == 0 { + // TODO: make the max restart configurable + if c := logger.Check(zapcore.InfoLevel, "restarting"); c != nil { + c.Write(zap.String("worker", thread.worker.fileName)) + } + metrics.StopWorker(thread.worker.fileName, StopReasonRestart) + } +} + +//export go_shutdown_woker_thread +func go_shutdown_woker_thread(threadIndex C.uintptr_t) { + workerShutdownWG.Done() +} + //export go_frankenphp_worker_handle_request_start func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { thread := phpThreads[threadIndex] - // we assign a worker to the thread if it doesn't have one already - if thread.worker == nil { - assignThreadToWorker(thread) + if !thread.isReady { + thread.isReady = true + workersReadyWG.Done() + metrics.ReadyWorker(thread.worker.fileName) } if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { From ad34140027c311b6e11d4e9755a691be98c7ace6 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 00:43:59 +0100 Subject: [PATCH 02/64] Moves code to separate file. --- frankenphp.c | 18 ++++---- frankenphp.go | 65 +++++++-------------------- php_thread.go | 19 -------- php_thread_test.go | 16 +------ php_threads.go | 108 +++++++++++++++++++++++++++++++++++++++++++++ worker.go | 36 ++++++--------- 6 files changed, 148 insertions(+), 114 deletions(-) create mode 100644 php_threads.go diff --git a/frankenphp.c b/frankenphp.c index ce71f7bec..93c283e3d 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -855,8 +855,8 @@ static void init_php_thread(void *arg) { local_ctx = malloc(sizeof(frankenphp_server_context)); } static void shutdown_php_thread(void) { - //free(local_ctx); - //local_ctx = NULL; + free(local_ctx); + local_ctx = NULL; #ifdef ZTS ts_free_thread(); #endif @@ -870,6 +870,7 @@ static void *php_thread(void *arg) { } shutdown_php_thread(); + go_shutdown_php_thread(thread_index); return NULL; } @@ -882,12 +883,12 @@ static void *php_worker_thread(void *arg) { if (script_name == NULL) { break; } - frankenphp_execute_script(script_name); - go_after_worker_script(thread_index); + int exit_status = frankenphp_execute_script(script_name); + go_after_worker_script(thread_index, exit_status); } shutdown_php_thread(); - go_shutdown_woker_thread(thread_index); + go_shutdown_worker_thread(thread_index); return NULL; } @@ -939,7 +940,7 @@ static void *php_main(void *arg) { frankenphp_sapi_module.startup(&frankenphp_sapi_module); - go_listen_for_shutdown(); + go_main_thread_is_ready(); /* channel closed, shutdown gracefully */ frankenphp_sapi_module.shutdown(&frankenphp_sapi_module); @@ -955,9 +956,7 @@ static void *php_main(void *arg) { frankenphp_sapi_module.ini_entries = NULL; } #endif - - go_shutdown(); - + go_shutdown_main_thread(); return NULL; } @@ -966,7 +965,6 @@ int frankenphp_new_main_thread(int num_threads) { if (pthread_create(&thread, NULL, &php_main, (void *)(intptr_t)num_threads) != 0) { - go_shutdown(); return -1; } return pthread_detach(thread); diff --git a/frankenphp.go b/frankenphp.go index 5bbbaeae0..f53870cbb 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -65,8 +65,6 @@ var ( requestChan chan *http.Request done chan struct{} - mainThreadWG sync.WaitGroup - shutdownWG sync.WaitGroup loggerMu sync.RWMutex logger *zap.Logger @@ -332,16 +330,19 @@ func Init(options ...Option) error { logger.Warn(`ZTS is not enabled, only 1 thread will be available, recompile PHP using the "--enable-zts" configuration option or performance will be degraded`) } - shutdownWG.Add(1) done = make(chan struct{}) requestChan = make(chan *http.Request) - initPHPThreads(opt.numThreads) + if err:= initPHPThreads(opt.numThreads); err != nil { + return err + } - startMainThread(opt.numThreads) + totalWorkers := 0 + for _, w := range opt.workers { + totalWorkers += w.num + } - // TODO: calc num threads - for i := 0; i < 1; i++ { - if err := startNewThread(); err != nil { + for i := 0; i < opt.numThreads - totalWorkers; i++ { + if err := startNewPHPThread(); err != nil { return err } } @@ -349,6 +350,7 @@ func Init(options ...Option) error { if err := initWorkers(opt.workers); err != nil { return err } + readyWG.Wait() if err := restartWorkersOnFileChanges(opt.workers); err != nil { return err @@ -369,7 +371,7 @@ func Init(options ...Option) error { // Shutdown stops the workers and the PHP runtime. func Shutdown() { drainWorkers() - drainThreads() + drainPHPThreads() metrics.Shutdown() requestChan = nil @@ -381,35 +383,6 @@ func Shutdown() { logger.Debug("FrankenPHP shut down") } -//export go_shutdown -func go_shutdown() { - shutdownWG.Done() -} - -func drainThreads() { - close(done) - shutdownWG.Wait() - phpThreads = nil -} - -func startMainThread(numThreads int) error { - mainThreadWG.Add(1) - if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { - return MainThreadCreationError - } - mainThreadWG.Wait() - return nil -} - -func startNewThread() error { - thread := getInactiveThread() - thread.isActive = true - if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("error creating thread %d", thread.threadIndex) - } - return nil -} - func getLogger() *zap.Logger { loggerMu.RLock() defer loggerMu.RUnlock() @@ -486,9 +459,6 @@ func updateServerContext(request *http.Request, create bool, isWorkerRequest boo // ServeHTTP executes a PHP script according to the given context. func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error { - shutdownWG.Add(1) - defer shutdownWG.Done() - fc, ok := FromContext(request.Context()) if !ok { return InvalidRequestError @@ -529,14 +499,6 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error return nil } -//export go_listen_for_shutdown -func go_listen_for_shutdown(){ - mainThreadWG.Done() - select{ - case <-done: - } -} - //export go_putenv func go_putenv(str *C.char, length C.int) C.bool { // Create a byte slice from C string with a specified length @@ -609,6 +571,11 @@ func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string //export go_handle_request func go_handle_request(threadIndex C.uintptr_t) bool { + thread := phpThreads[threadIndex] + if !thread.isReady { + thread.isReady = true + readyWG.Done() + } select { case <-done: return false diff --git a/php_thread.go b/php_thread.go index 608040b93..5611a1d04 100644 --- a/php_thread.go +++ b/php_thread.go @@ -7,8 +7,6 @@ import ( "runtime" ) -var phpThreads []*phpThread - type phpThread struct { runtime.Pinner @@ -20,23 +18,6 @@ type phpThread struct { threadIndex int } -func initPHPThreads(numThreads int) { - phpThreads = make([]*phpThread, 0, numThreads) - for i := 0; i < numThreads; i++ { - phpThreads = append(phpThreads, &phpThread{threadIndex: i}) - } -} - -func getInactiveThread() *phpThread { - for _, thread := range phpThreads { - if !thread.isActive { - return thread - } - } - - return nil -} - func (thread phpThread) getActiveRequest() *http.Request { if thread.workerRequest != nil { return thread.workerRequest diff --git a/php_thread_test.go b/php_thread_test.go index 63afe4d89..eba873d5b 100644 --- a/php_thread_test.go +++ b/php_thread_test.go @@ -7,20 +7,9 @@ import ( "github.com/stretchr/testify/assert" ) -func TestInitializeTwoPhpThreadsWithoutRequests(t *testing.T) { - initPHPThreads(2) - - assert.Len(t, phpThreads, 2) - assert.NotNil(t, phpThreads[0]) - assert.NotNil(t, phpThreads[1]) - assert.Nil(t, phpThreads[0].mainRequest) - assert.Nil(t, phpThreads[0].workerRequest) -} - func TestMainRequestIsActiveRequest(t *testing.T) { mainRequest := &http.Request{} - initPHPThreads(1) - thread := phpThreads[0] + thread := phpThread{} thread.mainRequest = mainRequest @@ -30,8 +19,7 @@ func TestMainRequestIsActiveRequest(t *testing.T) { func TestWorkerRequestIsActiveRequest(t *testing.T) { mainRequest := &http.Request{} workerRequest := &http.Request{} - initPHPThreads(1) - thread := phpThreads[0] + thread := phpThread{} thread.mainRequest = mainRequest thread.workerRequest = workerRequest diff --git a/php_threads.go b/php_threads.go new file mode 100644 index 000000000..417bfa75e --- /dev/null +++ b/php_threads.go @@ -0,0 +1,108 @@ +package frankenphp + +// #include +// #include "frankenphp.h" +import "C" +import ( + "fmt" + "sync" +) + +var ( + phpThreads []*phpThread + mainThreadWG sync.WaitGroup + terminationWG sync.WaitGroup + mainThreadShutdownWG sync.WaitGroup + readyWG sync.WaitGroup + shutdownWG sync.WaitGroup +) + +// reserve a fixed number of PHP threads on the go side +func initPHPThreads(numThreads int) error { + phpThreads = make([]*phpThread, numThreads) + for i := 0; i < numThreads; i++ { + phpThreads[i] = &phpThread{threadIndex: i} + } + return startMainThread(numThreads) +} + +func drainPHPThreads() { + close(done) + shutdownWG.Wait() + phpThreads = nil + mainThreadShutdownWG.Done() + terminationWG.Wait() +} + +func startMainThread(numThreads int) error { + mainThreadWG.Add(1) + mainThreadShutdownWG.Add(1) + terminationWG.Add(1) + if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { + return MainThreadCreationError + } + mainThreadWG.Wait() + return nil +} + +func startNewPHPThread() error { + readyWG.Add(1) + shutdownWG.Add(1) + thread := getInactiveThread() + thread.isActive = true + if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("error creating thread %d", thread.threadIndex) + } + return nil +} + +func startNewWorkerThread(worker *worker) error { + workersReadyWG.Add(1) + workerShutdownWG.Add(1) + thread := getInactiveThread() + thread.worker = worker + thread.isActive = true + if C.frankenphp_new_worker_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("failed to create worker thread") + } + + return nil +} + +func getInactiveThread() *phpThread { + for _, thread := range phpThreads { + if !thread.isActive { + return thread + } + } + + return nil +} + +//export go_main_thread_is_ready +func go_main_thread_is_ready(){ + mainThreadWG.Done() + mainThreadShutdownWG.Wait() +} + +//export go_shutdown_main_thread +func go_shutdown_main_thread(){ + terminationWG.Done() +} + +//export go_shutdown_php_thread +func go_shutdown_php_thread(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + thread.Unpin() + thread.isActive = false + shutdownWG.Done() +} + +//export go_shutdown_worker_thread +func go_shutdown_worker_thread(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + thread.Unpin() + thread.isActive = false + thread.worker = nil + workerShutdownWG.Done() +} \ No newline at end of file diff --git a/worker.go b/worker.go index 0dd71d1fb..eaa8e2a6d 100644 --- a/worker.go +++ b/worker.go @@ -48,7 +48,7 @@ func initWorkers(opt []workerOpt) error { return err } for i := 0; i < worker.num; i++ { - if err := worker.startNewThread(nil); err != nil { + if err := startNewWorkerThread(worker); err != nil { return err } } @@ -83,20 +83,7 @@ func newWorker(o workerOpt) (*worker, error) { return w, nil } -func (worker *worker) startNewThread(r *http.Request) error { - workersReadyWG.Add(1) - workerShutdownWG.Add(1) - thread := getInactiveThread() - thread.worker = worker - thread.isActive = true - if C.frankenphp_new_worker_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("failed to create worker thread") - } - - return nil -} - -func (worker *worker) startNewWorkerThread() { +func (worker *worker) asdasd() { workerShutdownWG.Add(1) defer workerShutdownWG.Done() @@ -289,10 +276,14 @@ func go_before_worker_script(threadIndex C.uintptr_t) *C.char { } //export go_after_worker_script -func go_after_worker_script(threadIndex C.uintptr_t) { +func go_after_worker_script(threadIndex C.uintptr_t, exitStatus C.int) { thread := phpThreads[threadIndex] fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + if fc.exitStatus < 0 { + panic(ScriptExecutionError) + } // on exit status 0 we just run the worker script again if fc.exitStatus == 0 { // TODO: make the max restart configurable @@ -300,12 +291,14 @@ func go_after_worker_script(threadIndex C.uintptr_t) { c.Write(zap.String("worker", thread.worker.fileName)) } metrics.StopWorker(thread.worker.fileName, StopReasonRestart) + return + } else { + time.Sleep(1 * time.Millisecond) + logger.Error("worker script exited with non-zero status") } -} - -//export go_shutdown_woker_thread -func go_shutdown_woker_thread(threadIndex C.uintptr_t) { - workerShutdownWG.Done() + maybeCloseContext(fc) + thread.mainRequest = nil + thread.Unpin() } //export go_frankenphp_worker_handle_request_start @@ -328,7 +321,6 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) } - thread.worker = nil executePHPFunction("opcache_reset") return C.bool(false) From 89b211d678e328a7de4995406119a781be90e80a Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 12:53:31 +0100 Subject: [PATCH 03/64] Cleans up the exponential backoff. --- exponential_backoff.go | 60 +++++++++++ frankenphp.go | 13 ++- php_thread.go | 21 +++- php_threads.go | 50 ++++----- worker.go | 232 +++++++++++------------------------------ 5 files changed, 170 insertions(+), 206 deletions(-) create mode 100644 exponential_backoff.go diff --git a/exponential_backoff.go b/exponential_backoff.go new file mode 100644 index 000000000..359e2bd4f --- /dev/null +++ b/exponential_backoff.go @@ -0,0 +1,60 @@ +package frankenphp + +import ( + "sync" + "time" +) + +const maxBackoff = 1 * time.Second +const minBackoff = 100 * time.Millisecond +const maxConsecutiveFailures = 6 + +type exponentialBackoff struct { + backoff time.Duration + failureCount int + mu sync.RWMutex + upFunc sync.Once +} + +func newExponentialBackoff() *exponentialBackoff { + return &exponentialBackoff{backoff: minBackoff} +} + +func (e *exponentialBackoff) reset() { + e.mu.Lock() + e.upFunc = sync.Once{} + wait := e.backoff * 2 + e.mu.Unlock() + go func() { + time.Sleep(wait) + e.mu.Lock() + defer e.mu.Unlock() + e.upFunc.Do(func() { + // if we come back to a stable state, reset the failure count + if e.backoff == minBackoff { + e.failureCount = 0 + } + + // earn back the backoff over time + if e.failureCount > 0 { + e.backoff = max(e.backoff/2, minBackoff) + } + }) + }() +} + +func (e *exponentialBackoff) trigger(onMaxFailures func(failureCount int)) { + e.mu.RLock() + e.upFunc.Do(func() { + if e.failureCount >= maxConsecutiveFailures { + onMaxFailures(e.failureCount) + } + e.failureCount += 1 + }) + wait := e.backoff + e.mu.RUnlock() + time.Sleep(wait) + e.mu.Lock() + e.backoff = min(e.backoff*2, maxBackoff) + e.mu.Unlock() +} diff --git a/frankenphp.go b/frankenphp.go index f53870cbb..df8d99af6 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -332,7 +332,7 @@ func Init(options ...Option) error { done = make(chan struct{}) requestChan = make(chan *http.Request) - if err:= initPHPThreads(opt.numThreads); err != nil { + if err := initPHPThreads(opt.numThreads); err != nil { return err } @@ -341,7 +341,7 @@ func Init(options ...Option) error { totalWorkers += w.num } - for i := 0; i < opt.numThreads - totalWorkers; i++ { + for i := 0; i < opt.numThreads-totalWorkers; i++ { if err := startNewPHPThread(); err != nil { return err } @@ -350,7 +350,9 @@ func Init(options ...Option) error { if err := initWorkers(opt.workers); err != nil { return err } - readyWG.Wait() + + // wait for all regular and worker threads to be ready for requests + threadsReadyWG.Wait() if err := restartWorkersOnFileChanges(opt.workers); err != nil { return err @@ -572,10 +574,7 @@ func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string //export go_handle_request func go_handle_request(threadIndex C.uintptr_t) bool { thread := phpThreads[threadIndex] - if !thread.isReady { - thread.isReady = true - readyWG.Done() - } + thread.setReadyForRequests() select { case <-done: return false diff --git a/php_thread.go b/php_thread.go index 5611a1d04..811c7a677 100644 --- a/php_thread.go +++ b/php_thread.go @@ -13,9 +13,10 @@ type phpThread struct { mainRequest *http.Request workerRequest *http.Request worker *worker - isActive bool - isReady bool - threadIndex int + isActive bool // whether the thread is currently running + isReady bool // whether the thread is ready to accept requests + threadIndex int // the index of the thread in the phpThreads slice + backoff *exponentialBackoff // backoff for worker failures } func (thread phpThread) getActiveRequest() *http.Request { @@ -25,3 +26,17 @@ func (thread phpThread) getActiveRequest() *http.Request { return thread.mainRequest } + +func (thread *phpThread) setReadyForRequests() { + if thread.isReady { + return + } + + thread.isReady = true + threadsReadyWG.Done() + if thread.worker != nil { + metrics.ReadyWorker(thread.worker.fileName) + } +} + + diff --git a/php_threads.go b/php_threads.go index 417bfa75e..55e19f53f 100644 --- a/php_threads.go +++ b/php_threads.go @@ -9,12 +9,11 @@ import ( ) var ( - phpThreads []*phpThread - mainThreadWG sync.WaitGroup - terminationWG sync.WaitGroup + phpThreads []*phpThread + terminationWG sync.WaitGroup mainThreadShutdownWG sync.WaitGroup - readyWG sync.WaitGroup - shutdownWG sync.WaitGroup + threadsReadyWG sync.WaitGroup + shutdownWG sync.WaitGroup ) // reserve a fixed number of PHP threads on the go side @@ -35,18 +34,18 @@ func drainPHPThreads() { } func startMainThread(numThreads int) error { - mainThreadWG.Add(1) + threadsReadyWG.Add(1) mainThreadShutdownWG.Add(1) terminationWG.Add(1) - if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { - return MainThreadCreationError - } - mainThreadWG.Wait() - return nil + if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { + return MainThreadCreationError + } + threadsReadyWG.Wait() + return nil } func startNewPHPThread() error { - readyWG.Add(1) + threadsReadyWG.Add(1) shutdownWG.Add(1) thread := getInactiveThread() thread.isActive = true @@ -57,14 +56,15 @@ func startNewPHPThread() error { } func startNewWorkerThread(worker *worker) error { - workersReadyWG.Add(1) - workerShutdownWG.Add(1) + threadsReadyWG.Add(1) + workerShutdownWG.Add(1) thread := getInactiveThread() - thread.worker = worker - thread.isActive = true - if C.frankenphp_new_worker_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("failed to create worker thread") - } + thread.worker = worker + thread.backoff = newExponentialBackoff() + thread.isActive = true + if C.frankenphp_new_worker_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("failed to create worker thread") + } return nil } @@ -80,13 +80,13 @@ func getInactiveThread() *phpThread { } //export go_main_thread_is_ready -func go_main_thread_is_ready(){ - mainThreadWG.Done() +func go_main_thread_is_ready() { + threadsReadyWG.Done() mainThreadShutdownWG.Wait() } //export go_shutdown_main_thread -func go_shutdown_main_thread(){ +func go_shutdown_main_thread() { terminationWG.Done() } @@ -95,6 +95,7 @@ func go_shutdown_php_thread(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] thread.Unpin() thread.isActive = false + thread.isReady = false shutdownWG.Done() } @@ -103,6 +104,7 @@ func go_shutdown_worker_thread(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] thread.Unpin() thread.isActive = false + thread.isReady = false thread.worker = nil - workerShutdownWG.Done() -} \ No newline at end of file + workerShutdownWG.Done() +} diff --git a/worker.go b/worker.go index eaa8e2a6d..96acc82c7 100644 --- a/worker.go +++ b/worker.go @@ -9,7 +9,6 @@ import ( "path/filepath" "sync" "sync/atomic" - "time" "github.com/dunglas/frankenphp/internal/watcher" "go.uber.org/zap" @@ -23,15 +22,9 @@ type worker struct { requestChan chan *http.Request } -const maxWorkerErrorBackoff = 1 * time.Second -const minWorkerErrorBackoff = 100 * time.Millisecond -const maxWorkerConsecutiveFailures = 6 - var ( watcherIsEnabled bool - workersReadyWG sync.WaitGroup workerShutdownWG sync.WaitGroup - workersAreReady atomic.Bool workersAreDone atomic.Bool workersDone chan interface{} workers = make(map[string]*worker) @@ -39,7 +32,6 @@ var ( func initWorkers(opt []workerOpt) error { workersDone = make(chan interface{}) - workersAreReady.Store(false) workersAreDone.Store(false) for _, o := range opt { @@ -54,9 +46,6 @@ func initWorkers(opt []workerOpt) error { } } - workersReadyWG.Wait() - workersAreReady.Store(true) - return nil } @@ -83,113 +72,6 @@ func newWorker(o workerOpt) (*worker, error) { return w, nil } -func (worker *worker) asdasd() { - workerShutdownWG.Add(1) - defer workerShutdownWG.Done() - - backoff := minWorkerErrorBackoff - failureCount := 0 - backingOffLock := sync.RWMutex{} - - for { - - // if the worker can stay up longer than backoff*2, it is probably an application error - upFunc := sync.Once{} - go func() { - backingOffLock.RLock() - wait := backoff * 2 - backingOffLock.RUnlock() - time.Sleep(wait) - upFunc.Do(func() { - backingOffLock.Lock() - defer backingOffLock.Unlock() - // if we come back to a stable state, reset the failure count - if backoff == minWorkerErrorBackoff { - failureCount = 0 - } - - // earn back the backoff over time - if failureCount > 0 { - backoff = max(backoff/2, 100*time.Millisecond) - } - }) - }() - - metrics.StartWorker(worker.fileName) - - // Create main dummy request - r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) - if err != nil { - panic(err) - } - - r, err = NewRequestWithContext( - r, - WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), - WithRequestPreparedEnv(worker.env), - ) - if err != nil { - panic(err) - } - - if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { - c.Write(zap.String("worker", worker.fileName), zap.Int("num", worker.num)) - } - - if err := ServeHTTP(nil, r); err != nil { - panic(err) - } - - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - - // if we are done, exit the loop that restarts the worker script - if workersAreDone.Load() { - break - } - - // on exit status 0 we just run the worker script again - if fc.exitStatus == 0 { - // TODO: make the max restart configurable - if c := logger.Check(zapcore.InfoLevel, "restarting"); c != nil { - c.Write(zap.String("worker", worker.fileName)) - } - metrics.StopWorker(worker.fileName, StopReasonRestart) - continue - } - - // on exit status 1 we log the error and apply an exponential backoff when restarting - upFunc.Do(func() { - backingOffLock.Lock() - defer backingOffLock.Unlock() - // if we end up here, the worker has not been up for backoff*2 - // this is probably due to a syntax error or another fatal error - if failureCount >= maxWorkerConsecutiveFailures { - if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) - } - logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) - } - failureCount += 1 - }) - backingOffLock.RLock() - wait := backoff - backingOffLock.RUnlock() - time.Sleep(wait) - backingOffLock.Lock() - backoff *= 2 - backoff = min(backoff, maxWorkerErrorBackoff) - backingOffLock.Unlock() - metrics.StopWorker(worker.fileName, StopReasonCrash) - } - - metrics.StopWorker(worker.fileName, StopReasonShutdown) - - // TODO: check if the termination is expected - if c := logger.Check(zapcore.DebugLevel, "terminated"); c != nil { - c.Write(zap.String("worker", worker.fileName)) - } -} - func stopWorkers() { workersAreDone.Store(true) close(workersDone) @@ -232,13 +114,6 @@ func restartWorkers(workerOpts []workerOpt) { logger.Info("workers restarted successfully") } -func assignThreadToWorker(thread *phpThread) { - metrics.ReadyWorker(thread.worker.fileName) - thread.isReady = true - workersReadyWG.Done() - // TODO: we can also store all threads assigned to the worker if needed -} - //export go_before_worker_script func go_before_worker_script(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] @@ -246,32 +121,37 @@ func go_before_worker_script(threadIndex C.uintptr_t) *C.char { // if we are done, exit the loop that restarts the worker script if workersAreDone.Load() { - return nil - } + return nil + } + + // if we are restarting the worker, reset the exponential failure backoff + thread.backoff.reset() metrics.StartWorker(worker.fileName) - // Create main dummy request - r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) - if err != nil { - panic(err) - } - - r, err = NewRequestWithContext( - r, - WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), - WithRequestPreparedEnv(worker.env), - ) - if err != nil { - panic(err) - } - thread.mainRequest = r - if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { - c.Write(zap.String("worker", worker.fileName), zap.Int("num", worker.num)) - } - - if err := updateServerContext(r, true, false); err != nil { - panic(err) - } + // Create a dummy request to set up the worker + r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) + if err != nil { + panic(err) + } + + r, err = NewRequestWithContext( + r, + WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), + WithRequestPreparedEnv(worker.env), + ) + if err != nil { + panic(err) + } + + if err := updateServerContext(r, true, false); err != nil { + panic(err) + } + + thread.mainRequest = r + if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { + c.Write(zap.String("worker", worker.fileName), zap.Int("num", worker.num)) + } + return C.CString(worker.fileName) } @@ -282,34 +162,42 @@ func go_after_worker_script(threadIndex C.uintptr_t, exitStatus C.int) { fc.exitStatus = exitStatus if fc.exitStatus < 0 { - panic(ScriptExecutionError) - } - // on exit status 0 we just run the worker script again - if fc.exitStatus == 0 { - // TODO: make the max restart configurable - if c := logger.Check(zapcore.InfoLevel, "restarting"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName)) - } - metrics.StopWorker(thread.worker.fileName, StopReasonRestart) - return - } else { - time.Sleep(1 * time.Millisecond) - logger.Error("worker script exited with non-zero status") - } - maybeCloseContext(fc) - thread.mainRequest = nil - thread.Unpin() + panic(ScriptExecutionError) + } + + defer func() { + maybeCloseContext(fc) + thread.mainRequest = nil + thread.Unpin() + }() + + // on exit status 0 we just run the worker script again + if fc.exitStatus == 0 { + // TODO: make the max restart configurable + metrics.StopWorker(thread.worker.fileName, StopReasonRestart) + + if c := logger.Check(zapcore.InfoLevel, "restarting"); c != nil { + c.Write(zap.String("worker", thread.worker.fileName)) + } + return + } + + // on exit status 1 we apply an exponential backoff when restarting + metrics.StopWorker(thread.worker.fileName, StopReasonCrash) + thread.backoff.trigger(func(failureCount int) { + // if we end up here, the worker has not been up for backoff*2 + // this is probably due to a syntax error or another fatal error + if !watcherIsEnabled { + panic(fmt.Errorf("workers %q: too many consecutive failures", thread.worker.fileName)) + } + logger.Warn("many consecutive worker failures", zap.String("worker", thread.worker.fileName), zap.Int("failures", failureCount)) + }) } //export go_frankenphp_worker_handle_request_start func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { thread := phpThreads[threadIndex] - - if !thread.isReady { - thread.isReady = true - workersReadyWG.Done() - metrics.ReadyWorker(thread.worker.fileName) - } + thread.setReadyForRequests() if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) From 7d2ab8cc99af0bd992be3cdabf8c190e7768f29f Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 14:28:52 +0100 Subject: [PATCH 04/64] Initial working implementation. --- frankenphp.go | 2 -- php_threads.go | 6 +++-- php_threads_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 php_threads_test.go diff --git a/frankenphp.go b/frankenphp.go index df8d99af6..765c48784 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -64,7 +64,6 @@ var ( ScriptExecutionError = errors.New("error during PHP script execution") requestChan chan *http.Request - done chan struct{} loggerMu sync.RWMutex logger *zap.Logger @@ -330,7 +329,6 @@ func Init(options ...Option) error { logger.Warn(`ZTS is not enabled, only 1 thread will be available, recompile PHP using the "--enable-zts" configuration option or performance will be degraded`) } - done = make(chan struct{}) requestChan = make(chan *http.Request) if err := initPHPThreads(opt.numThreads); err != nil { return err diff --git a/php_threads.go b/php_threads.go index 55e19f53f..14718ed41 100644 --- a/php_threads.go +++ b/php_threads.go @@ -12,12 +12,14 @@ var ( phpThreads []*phpThread terminationWG sync.WaitGroup mainThreadShutdownWG sync.WaitGroup - threadsReadyWG sync.WaitGroup + threadsReadyWG sync.WaitGroup shutdownWG sync.WaitGroup + done chan struct{} ) // reserve a fixed number of PHP threads on the go side func initPHPThreads(numThreads int) error { + done = make(chan struct{}) phpThreads = make([]*phpThread, numThreads) for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{threadIndex: i} @@ -28,9 +30,9 @@ func initPHPThreads(numThreads int) error { func drainPHPThreads() { close(done) shutdownWG.Wait() - phpThreads = nil mainThreadShutdownWG.Done() terminationWG.Wait() + phpThreads = nil } func startMainThread(numThreads int) error { diff --git a/php_threads_test.go b/php_threads_test.go new file mode 100644 index 000000000..912485309 --- /dev/null +++ b/php_threads_test.go @@ -0,0 +1,60 @@ +package frankenphp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func ATestStartAndStopTheMainThread(t *testing.T) { + logger = zap.NewNop() + initPHPThreads(1) // reserve 1 thread + + assert.Equal(t, 1, len(phpThreads)) + assert.Equal(t, 0, phpThreads[0].threadIndex) + assert.False(t, phpThreads[0].isActive) + assert.False(t, phpThreads[0].isReady) + assert.Nil(t, phpThreads[0].worker) + + drainPHPThreads() + assert.Nil(t, phpThreads) +} + +func ATestStartAndStopARegularThread(t *testing.T) { + logger = zap.NewNop() // the logger needs to not be nil + initPHPThreads(1) // reserve 1 thread + + startNewPHPThread() + threadsReadyWG.Wait() + + assert.Equal(t, 1, len(phpThreads)) + assert.True(t, phpThreads[0].isActive) + assert.True(t, phpThreads[0].isReady) + assert.Nil(t, phpThreads[0].worker) + + drainPHPThreads() + assert.Nil(t, phpThreads) +} + +func ATestStartAndStopAWorkerThread(t *testing.T) { + logger = zap.NewNop() // the logger needs to not be nil + initPHPThreads(1) // reserve 1 thread + + initWorkers([]workerOpt{workerOpt { + fileName: "testdata/worker.php", + num: 1, + env: make(map[string]string, 0), + watch: make([]string, 0), + }}) + threadsReadyWG.Wait() + + assert.Equal(t, 1, len(phpThreads)) + assert.True(t, phpThreads[0].isActive) + assert.True(t, phpThreads[0].isReady) + assert.NotNil(t, phpThreads[0].worker) + + drainPHPThreads() + assert.Nil(t, phpThreads) +} + From f7e7d41f8766e7c64f6f1624957300c3f4b41f2f Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 21:27:43 +0100 Subject: [PATCH 05/64] Refactors php threads to take callbacks. --- frankenphp.c | 67 +++++++------------------- frankenphp.go | 37 +++++++-------- frankenphp.h | 1 - php_thread.go | 65 ++++++++++++++++++++++---- php_threads.go | 57 +++-------------------- php_threads_test.go | 111 ++++++++++++++++++++++++++++++++------------ testdata/sleep.php | 4 ++ worker.go | 61 +++++++++++++++--------- 8 files changed, 217 insertions(+), 186 deletions(-) create mode 100644 testdata/sleep.php diff --git a/frankenphp.c b/frankenphp.c index 93c283e3d..988404e81 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -838,7 +838,7 @@ static void set_thread_name(char *thread_name) { #endif } -static void init_php_thread(void *arg) { +static void *php_thread(void *arg) { thread_index = (uintptr_t)arg; char thread_name[16] = {0}; snprintf(thread_name, 16, "php-%" PRIxPTR, thread_index); @@ -851,44 +851,20 @@ static void init_php_thread(void *arg) { ZEND_TSRMLS_CACHE_UPDATE(); #endif #endif - local_ctx = malloc(sizeof(frankenphp_server_context)); -} -static void shutdown_php_thread(void) { - free(local_ctx); - local_ctx = NULL; -#ifdef ZTS - ts_free_thread(); -#endif -} -static void *php_thread(void *arg) { - init_php_thread(arg); + go_frankenphp_on_thread_startup(thread_index); - // handle requests until the channel is closed - while (go_handle_request(thread_index)) { + // perform work until go signals to stop + while (go_frankenphp_on_thread_work(thread_index)) { } - shutdown_php_thread(); - go_shutdown_php_thread(thread_index); - return NULL; -} +#ifdef ZTS + ts_free_thread(); +#endif -static void *php_worker_thread(void *arg) { - init_php_thread(arg); - - // run the loop that executes the worker script - while (true) { - char *script_name = go_before_worker_script(thread_index); - if (script_name == NULL) { - break; - } - int exit_status = frankenphp_execute_script(script_name); - go_after_worker_script(thread_index, exit_status); - } + go_frankenphp_on_thread_shutdown(thread_index); - shutdown_php_thread(); - go_shutdown_worker_thread(thread_index); return NULL; } @@ -940,7 +916,7 @@ static void *php_main(void *arg) { frankenphp_sapi_module.startup(&frankenphp_sapi_module); - go_main_thread_is_ready(); + go_frankenphp_main_thread_is_ready(); /* channel closed, shutdown gracefully */ frankenphp_sapi_module.shutdown(&frankenphp_sapi_module); @@ -956,7 +932,7 @@ static void *php_main(void *arg) { frankenphp_sapi_module.ini_entries = NULL; } #endif - go_shutdown_main_thread(); + go_frankenphp_shutdown_main_thread(); return NULL; } @@ -970,22 +946,13 @@ int frankenphp_new_main_thread(int num_threads) { return pthread_detach(thread); } -int frankenphp_new_worker_thread(uintptr_t thread_index){ - pthread_t thread; - if (pthread_create(&thread, NULL, &php_worker_thread, (void *)thread_index) != 0){ - return 1; - } - pthread_detach(thread); - return 0; -} - -int frankenphp_new_php_thread(uintptr_t thread_index){ - pthread_t thread; - if (pthread_create(&thread, NULL, &php_thread, (void *)thread_index) != 0){ - return 1; - } - pthread_detach(thread); - return 0; +int frankenphp_new_php_thread(uintptr_t thread_index) { + pthread_t thread; + if (pthread_create(&thread, NULL, &php_thread, (void *)thread_index) != 0) { + return 1; + } + pthread_detach(thread); + return 0; } int frankenphp_request_startup() { diff --git a/frankenphp.go b/frankenphp.go index 765c48784..53bc9b2c3 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -340,7 +340,9 @@ func Init(options ...Option) error { } for i := 0; i < opt.numThreads-totalWorkers; i++ { - if err := startNewPHPThread(); err != nil { + thread := getInactivePHPThread() + thread.onWork = handleRequest + if err := thread.run(); err != nil { return err } } @@ -569,16 +571,12 @@ func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string return true, value // Return 1 to indicate success } -//export go_handle_request -func go_handle_request(threadIndex C.uintptr_t) bool { - thread := phpThreads[threadIndex] - thread.setReadyForRequests() +func handleRequest(thread *phpThread) bool { select { case <-done: return false case r := <-requestChan: - thread := phpThreads[threadIndex] thread.mainRequest = r fc, ok := FromContext(r.Context()) @@ -595,11 +593,7 @@ func go_handle_request(threadIndex C.uintptr_t) bool { panic(err) } - // scriptFilename is freed in frankenphp_execute_script() - fc.exitStatus = C.frankenphp_execute_script(C.CString(fc.scriptFilename)) - if fc.exitStatus < 0 { - panic(ScriptExecutionError) - } + fc.exitStatus = executeScriptCGI(fc.scriptFilename) return true } @@ -880,6 +874,15 @@ func go_log(message *C.char, level C.int) { } } +func executeScriptCGI(script string) C.int { + // scriptFilename is freed in frankenphp_execute_script() + exitStatus := C.frankenphp_execute_script(C.CString(script)) + if exitStatus < 0 { + panic(ScriptExecutionError) + } + return exitStatus +} + // ExecuteScriptCLI executes the PHP script passed as parameter. // It returns the exit status code of the script. func ExecuteScriptCLI(script string, args []string) int { @@ -907,19 +910,11 @@ func freeArgs(argv []*C.char) { } } -func executePHPFunction(functionName string) { +func executePHPFunction(functionName string) bool { cFunctionName := C.CString(functionName) defer C.free(unsafe.Pointer(cFunctionName)) success := C.frankenphp_execute_php_function(cFunctionName) - if success == 1 { - if c := logger.Check(zapcore.DebugLevel, "php function call successful"); c != nil { - c.Write(zap.String("function", functionName)) - } - } else { - if c := logger.Check(zapcore.ErrorLevel, "php function call failed"); c != nil { - c.Write(zap.String("function", functionName)) - } - } + return success == 1 } diff --git a/frankenphp.h b/frankenphp.h index 7470ba00e..38d408fe6 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -42,7 +42,6 @@ frankenphp_config frankenphp_get_config(); int frankenphp_new_main_thread(int num_threads); int frankenphp_new_php_thread(uintptr_t thread_index); -int frankenphp_new_worker_thread(uintptr_t thread_index); int frankenphp_update_server_context( bool create, bool has_main_request, bool has_active_request, diff --git a/php_thread.go b/php_thread.go index 811c7a677..309107736 100644 --- a/php_thread.go +++ b/php_thread.go @@ -1,8 +1,11 @@ package frankenphp // #include +// #include +// #include "frankenphp.h" import "C" import ( + "fmt" "net/http" "runtime" ) @@ -13,10 +16,13 @@ type phpThread struct { mainRequest *http.Request workerRequest *http.Request worker *worker - isActive bool // whether the thread is currently running - isReady bool // whether the thread is ready to accept requests - threadIndex int // the index of the thread in the phpThreads slice - backoff *exponentialBackoff // backoff for worker failures + isActive bool // whether the thread is currently running + isReady bool // whether the thread is ready to accept requests + threadIndex int // the index of the thread in the phpThreads slice + onStartup func(*phpThread) // the function to run on startup + onWork func(*phpThread) bool // the function to run in the thread + onShutdown func(*phpThread) // the function to run after shutdown + backoff *exponentialBackoff // backoff for worker failures } func (thread phpThread) getActiveRequest() *http.Request { @@ -27,16 +33,55 @@ func (thread phpThread) getActiveRequest() *http.Request { return thread.mainRequest } -func (thread *phpThread) setReadyForRequests() { +func (thread *phpThread) run() error { + if thread.isActive { + return fmt.Errorf("thread is already running %d", thread.threadIndex) + } + threadsReadyWG.Add(1) + shutdownWG.Add(1) + thread.isActive = true + if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("error creating thread %d", thread.threadIndex) + } + return nil +} + +func (thread *phpThread) setReady() { if thread.isReady { return } - thread.isReady = true - threadsReadyWG.Done() - if thread.worker != nil { - metrics.ReadyWorker(thread.worker.fileName) - } + thread.isReady = true + threadsReadyWG.Done() + if thread.worker != nil { + metrics.ReadyWorker(thread.worker.fileName) + } } +//export go_frankenphp_on_thread_startup +func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + thread.isReady = true + if thread.onStartup != nil { + thread.onStartup(thread) + } + threadsReadyWG.Done() +} + +//export go_frankenphp_on_thread_work +func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { + thread := phpThreads[threadIndex] + return C.bool(thread.onWork(thread)) +} +//export go_frankenphp_on_thread_shutdown +func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + thread.isActive = false + thread.isReady = false + thread.Unpin() + if thread.onShutdown != nil { + thread.onShutdown(thread) + } + shutdownWG.Done() +} diff --git a/php_threads.go b/php_threads.go index 14718ed41..137eb4ee8 100644 --- a/php_threads.go +++ b/php_threads.go @@ -4,7 +4,6 @@ package frankenphp // #include "frankenphp.h" import "C" import ( - "fmt" "sync" ) @@ -14,7 +13,7 @@ var ( mainThreadShutdownWG sync.WaitGroup threadsReadyWG sync.WaitGroup shutdownWG sync.WaitGroup - done chan struct{} + done chan struct{} ) // reserve a fixed number of PHP threads on the go side @@ -46,32 +45,7 @@ func startMainThread(numThreads int) error { return nil } -func startNewPHPThread() error { - threadsReadyWG.Add(1) - shutdownWG.Add(1) - thread := getInactiveThread() - thread.isActive = true - if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("error creating thread %d", thread.threadIndex) - } - return nil -} - -func startNewWorkerThread(worker *worker) error { - threadsReadyWG.Add(1) - workerShutdownWG.Add(1) - thread := getInactiveThread() - thread.worker = worker - thread.backoff = newExponentialBackoff() - thread.isActive = true - if C.frankenphp_new_worker_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("failed to create worker thread") - } - - return nil -} - -func getInactiveThread() *phpThread { +func getInactivePHPThread() *phpThread { for _, thread := range phpThreads { if !thread.isActive { return thread @@ -81,32 +55,13 @@ func getInactiveThread() *phpThread { return nil } -//export go_main_thread_is_ready -func go_main_thread_is_ready() { +//export go_frankenphp_main_thread_is_ready +func go_frankenphp_main_thread_is_ready() { threadsReadyWG.Done() mainThreadShutdownWG.Wait() } -//export go_shutdown_main_thread -func go_shutdown_main_thread() { +//export go_frankenphp_shutdown_main_thread +func go_frankenphp_shutdown_main_thread() { terminationWG.Done() } - -//export go_shutdown_php_thread -func go_shutdown_php_thread(threadIndex C.uintptr_t) { - thread := phpThreads[threadIndex] - thread.Unpin() - thread.isActive = false - thread.isReady = false - shutdownWG.Done() -} - -//export go_shutdown_worker_thread -func go_shutdown_worker_thread(threadIndex C.uintptr_t) { - thread := phpThreads[threadIndex] - thread.Unpin() - thread.isActive = false - thread.isReady = false - thread.worker = nil - workerShutdownWG.Done() -} diff --git a/php_threads_test.go b/php_threads_test.go index 912485309..837486054 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -1,60 +1,111 @@ package frankenphp import ( + "net/http" + "path/filepath" + "sync" + "sync/atomic" "testing" "github.com/stretchr/testify/assert" "go.uber.org/zap" ) -func ATestStartAndStopTheMainThread(t *testing.T) { - logger = zap.NewNop() +func TestStartAndStopTheMainThread(t *testing.T) { initPHPThreads(1) // reserve 1 thread assert.Equal(t, 1, len(phpThreads)) assert.Equal(t, 0, phpThreads[0].threadIndex) - assert.False(t, phpThreads[0].isActive) - assert.False(t, phpThreads[0].isReady) - assert.Nil(t, phpThreads[0].worker) + assert.False(t, phpThreads[0].isActive) + assert.False(t, phpThreads[0].isReady) + assert.Nil(t, phpThreads[0].worker) drainPHPThreads() assert.Nil(t, phpThreads) } -func ATestStartAndStopARegularThread(t *testing.T) { - logger = zap.NewNop() // the logger needs to not be nil - initPHPThreads(1) // reserve 1 thread +// We'll start 100 threads and check that their hooks work correctly +// onStartup => before the thread is ready +// onWork => while the thread is working +// onShutdown => after the thread is done +func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { + numThreads := 100 + readyThreads := atomic.Uint64{} + finishedThreads := atomic.Uint64{} + workingThreads := atomic.Uint64{} + initPHPThreads(numThreads) + + for i := 0; i < numThreads; i++ { + newThread := getInactivePHPThread() + newThread.onStartup = func(thread *phpThread) { + if thread.threadIndex == newThread.threadIndex { + readyThreads.Add(1) + } + } + newThread.onWork = func(thread *phpThread) bool { + if thread.threadIndex == newThread.threadIndex { + workingThreads.Add(1) + } + return false // stop immediately + } + newThread.onShutdown = func(thread *phpThread) { + if thread.threadIndex == newThread.threadIndex { + finishedThreads.Add(1) + } + } + newThread.run() + } - startNewPHPThread() threadsReadyWG.Wait() - assert.Equal(t, 1, len(phpThreads)) - assert.True(t, phpThreads[0].isActive) - assert.True(t, phpThreads[0].isReady) - assert.Nil(t, phpThreads[0].worker) + assert.Equal(t, numThreads, int(readyThreads.Load())) drainPHPThreads() - assert.Nil(t, phpThreads) + + assert.Equal(t, numThreads, int(workingThreads.Load())) + assert.Equal(t, numThreads, int(finishedThreads.Load())) } -func ATestStartAndStopAWorkerThread(t *testing.T) { +// This test calls sleep() 10.000 times for 1ms (completes in ~200ms) +func TestSleep10000TimesIn100Threads(t *testing.T) { logger = zap.NewNop() // the logger needs to not be nil - initPHPThreads(1) // reserve 1 thread - - initWorkers([]workerOpt{workerOpt { - fileName: "testdata/worker.php", - num: 1, - env: make(map[string]string, 0), - watch: make([]string, 0), - }}) - threadsReadyWG.Wait() + numThreads := 100 + maxExecutions := 10000 + executionMutex := sync.Mutex{} + executionCount := 0 + scriptPath, _ := filepath.Abs("./testdata/sleep.php") + initPHPThreads(numThreads) - assert.Equal(t, 1, len(phpThreads)) - assert.True(t, phpThreads[0].isActive) - assert.True(t, phpThreads[0].isReady) - assert.NotNil(t, phpThreads[0].worker) + for i := 0; i < numThreads; i++ { + newThread := getInactivePHPThread() + + // fake a request on startup (like a worker would do) + newThread.onStartup = func(thread *phpThread) { + r, _ := http.NewRequest(http.MethodGet, "sleep.php", nil) + r, _ = NewRequestWithContext(r, WithRequestDocumentRoot("/", false)) + assert.NoError(t, updateServerContext(r, true, false)) + thread.mainRequest = r + } + + // execute the php script until we reach the maxExecutions + newThread.onWork = func(thread *phpThread) bool { + executionMutex.Lock() + if executionCount >= maxExecutions { + executionMutex.Unlock() + return false + } + executionCount++ + executionMutex.Unlock() + if int(executeScriptCGI(scriptPath)) != 0 { + return false + } + + return true + } + newThread.run() + } drainPHPThreads() - assert.Nil(t, phpThreads) -} + assert.Equal(t, maxExecutions, executionCount) +} diff --git a/testdata/sleep.php b/testdata/sleep.php new file mode 100644 index 000000000..1b1a66d02 --- /dev/null +++ b/testdata/sleep.php @@ -0,0 +1,4 @@ + Date: Sat, 2 Nov 2024 22:04:53 +0100 Subject: [PATCH 06/64] Cleanup. --- frankenphp.c | 4 ++-- frankenphp.go | 21 ++++++++------------- php_thread.go | 25 +++++++------------------ php_threads_test.go | 3 +-- worker.go | 33 ++++++++++++++++++--------------- 5 files changed, 36 insertions(+), 50 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 988404e81..42bdfca39 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -253,7 +253,7 @@ PHP_FUNCTION(frankenphp_finish_request) { /* {{{ */ php_header(); if (ctx->has_active_request) { - go_frankenphp_finish_request(thread_index, false); + go_frankenphp_finish_request_manually(thread_index); } ctx->finished = true; @@ -453,7 +453,7 @@ PHP_FUNCTION(frankenphp_handle_request) { frankenphp_worker_request_shutdown(); ctx->has_active_request = false; - go_frankenphp_finish_request(thread_index, true); + go_frankenphp_finish_worker_request(thread_index); RETURN_TRUE; } diff --git a/frankenphp.go b/frankenphp.go index 53bc9b2c3..1db0714bc 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -242,7 +242,7 @@ func Config() PHPConfig { // MaxThreads is internally used during tests. It is written to, but never read and may go away in the future. var MaxThreads int -func calculateMaxThreads(opt *opt) error { +func calculateMaxThreads(opt *opt) (int, int, error) { maxProcs := runtime.GOMAXPROCS(0) * 2 var numWorkers int @@ -264,13 +264,13 @@ func calculateMaxThreads(opt *opt) error { opt.numThreads = maxProcs } } else if opt.numThreads <= numWorkers { - return NotEnoughThreads + return opt.numThreads, numWorkers, NotEnoughThreads } metrics.TotalThreads(opt.numThreads) MaxThreads = opt.numThreads - return nil + return opt.numThreads, numWorkers, nil } // Init starts the PHP runtime and the configured workers. @@ -309,7 +309,7 @@ func Init(options ...Option) error { metrics = opt.metrics } - err := calculateMaxThreads(opt) + totalThreadCount, workerThreadCount, err := calculateMaxThreads(opt) if err != nil { return err } @@ -325,21 +325,16 @@ func Init(options ...Option) error { logger.Warn(`Zend Max Execution Timers are not enabled, timeouts (e.g. "max_execution_time") are disabled, recompile PHP with the "--enable-zend-max-execution-timers" configuration option to fix this issue`) } } else { - opt.numThreads = 1 + totalThreadCount = 1 logger.Warn(`ZTS is not enabled, only 1 thread will be available, recompile PHP using the "--enable-zts" configuration option or performance will be degraded`) } requestChan = make(chan *http.Request) - if err := initPHPThreads(opt.numThreads); err != nil { + if err := initPHPThreads(totalThreadCount); err != nil { return err } - totalWorkers := 0 - for _, w := range opt.workers { - totalWorkers += w.num - } - - for i := 0; i < opt.numThreads-totalWorkers; i++ { + for i := 0; i < totalThreadCount-workerThreadCount; i++ { thread := getInactivePHPThread() thread.onWork = handleRequest if err := thread.run(); err != nil { @@ -359,7 +354,7 @@ func Init(options ...Option) error { } if c := logger.Check(zapcore.InfoLevel, "FrankenPHP started 🐘"); c != nil { - c.Write(zap.String("php_version", Version().Version), zap.Int("num_threads", opt.numThreads)) + c.Write(zap.String("php_version", Version().Version), zap.Int("num_threads", totalThreadCount)) } if EmbeddedAppPath != "" { if c := logger.Check(zapcore.InfoLevel, "embedded PHP app 📦"); c != nil { diff --git a/php_thread.go b/php_thread.go index 309107736..351b7e356 100644 --- a/php_thread.go +++ b/php_thread.go @@ -16,11 +16,10 @@ type phpThread struct { mainRequest *http.Request workerRequest *http.Request worker *worker - isActive bool // whether the thread is currently running - isReady bool // whether the thread is ready to accept requests threadIndex int // the index of the thread in the phpThreads slice - onStartup func(*phpThread) // the function to run on startup - onWork func(*phpThread) bool // the function to run in the thread + isActive bool // whether the thread is currently running + onStartup func(*phpThread) // the function to run when ready + onWork func(*phpThread) bool // the function to run in a loop when ready onShutdown func(*phpThread) // the function to run after shutdown backoff *exponentialBackoff // backoff for worker failures } @@ -37,31 +36,22 @@ func (thread *phpThread) run() error { if thread.isActive { return fmt.Errorf("thread is already running %d", thread.threadIndex) } + if thread.onWork == nil { + return fmt.Errorf("thread.onWork must be defined %d", thread.threadIndex) + } threadsReadyWG.Add(1) shutdownWG.Add(1) thread.isActive = true if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { return fmt.Errorf("error creating thread %d", thread.threadIndex) } - return nil -} - -func (thread *phpThread) setReady() { - if thread.isReady { - return - } - thread.isReady = true - threadsReadyWG.Done() - if thread.worker != nil { - metrics.ReadyWorker(thread.worker.fileName) - } + return nil } //export go_frankenphp_on_thread_startup func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] - thread.isReady = true if thread.onStartup != nil { thread.onStartup(thread) } @@ -78,7 +68,6 @@ func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] thread.isActive = false - thread.isReady = false thread.Unpin() if thread.onShutdown != nil { thread.onShutdown(thread) diff --git a/php_threads_test.go b/php_threads_test.go index 837486054..3344e901f 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -17,7 +17,6 @@ func TestStartAndStopTheMainThread(t *testing.T) { assert.Equal(t, 1, len(phpThreads)) assert.Equal(t, 0, phpThreads[0].threadIndex) assert.False(t, phpThreads[0].isActive) - assert.False(t, phpThreads[0].isReady) assert.Nil(t, phpThreads[0].worker) drainPHPThreads() @@ -66,7 +65,7 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { assert.Equal(t, numThreads, int(finishedThreads.Load())) } -// This test calls sleep() 10.000 times for 1ms (completes in ~200ms) +// This test calls sleep() 10.000 times for 1ms in 100 PHP threads. func TestSleep10000TimesIn100Threads(t *testing.T) { logger = zap.NewNop() // the logger needs to not be nil numThreads := 100 diff --git a/worker.go b/worker.go index bffed0327..b4497de3e 100644 --- a/worker.go +++ b/worker.go @@ -210,7 +210,6 @@ func afterWorkerScript(thread *phpThread, exitStatus C.int) { //export go_frankenphp_worker_handle_request_start func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { thread := phpThreads[threadIndex] - thread.setReady() if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) @@ -247,28 +246,32 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { return C.bool(true) } -//export go_frankenphp_finish_request -func go_frankenphp_finish_request(threadIndex C.uintptr_t, isWorkerRequest bool) { +//export go_frankenphp_finish_worker_request +func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] r := thread.getActiveRequest() fc := r.Context().Value(contextKey).(*FrankenPHPContext) - - if isWorkerRequest { - thread.workerRequest = nil - } + thread.workerRequest = nil maybeCloseContext(fc) if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { - var fields []zap.Field - if isWorkerRequest { - fields = append(fields, zap.String("worker", fc.scriptFilename), zap.String("url", r.RequestURI)) - } else { - fields = append(fields, zap.String("url", r.RequestURI)) - } - - c.Write(fields...) + c.Write(zap.String("worker", fc.scriptFilename), zap.String("url", r.RequestURI)) } thread.Unpin() } + +// when frankenphp_finish_request() is directly called from PHP +// +//export go_frankenphp_finish_request_manually +func go_frankenphp_finish_request_manually(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + r := thread.getActiveRequest() + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + maybeCloseContext(fc) + + if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { + c.Write(zap.String("url", r.RequestURI)) + } +} From a9857dc82eeb7cfb69a8d818aabbf0645c3bba47 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 22:18:47 +0100 Subject: [PATCH 07/64] Cleanup. --- php_threads.go | 3 +-- worker.go | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/php_threads.go b/php_threads.go index 137eb4ee8..180594d66 100644 --- a/php_threads.go +++ b/php_threads.go @@ -51,8 +51,7 @@ func getInactivePHPThread() *phpThread { return thread } } - - return nil + panic("not enough threads reserved") } //export go_frankenphp_main_thread_is_ready diff --git a/worker.go b/worker.go index b4497de3e..1ba3e110c 100644 --- a/worker.go +++ b/worker.go @@ -75,16 +75,33 @@ func newWorker(o workerOpt) (*worker, error) { func startNewWorkerThread(worker *worker) error { workerShutdownWG.Add(1) thread := getInactivePHPThread() + + // onStartup => right before the thread is ready thread.onStartup = func(thread *phpThread) { thread.worker = worker metrics.ReadyWorker(worker.fileName) thread.backoff = newExponentialBackoff() } - thread.onWork = runWorkerScript + + // onWork => while the thread is working (in a loop) + thread.onWork = func(thread *phpThread) bool { + if workersAreDone.Load() { + return false + } + beforeWorkerScript(thread) + exitStatus := executeScriptCGI(thread.worker.fileName) + afterWorkerScript(thread, exitStatus) + + return true + } + + // onShutdown => after the thread is done thread.onShutdown = func(thread *phpThread) { thread.worker = nil + thread.backoff = nil workerShutdownWG.Done() } + return thread.run() } @@ -130,18 +147,6 @@ func restartWorkers(workerOpts []workerOpt) { logger.Info("workers restarted successfully") } -func runWorkerScript(thread *phpThread) bool { - // if workers are done, we stop the loop that runs the worker script - if workersAreDone.Load() { - return false - } - beforeWorkerScript(thread) - exitStatus := executeScriptCGI(thread.worker.fileName) - afterWorkerScript(thread, exitStatus) - - return true -} - func beforeWorkerScript(thread *phpThread) { worker := thread.worker From bac9555d91b5463fc315fa5936afc376d76eba47 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 22:20:54 +0100 Subject: [PATCH 08/64] Cleanup. --- php_threads_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php_threads_test.go b/php_threads_test.go index 3344e901f..3a074818a 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -86,7 +86,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { thread.mainRequest = r } - // execute the php script until we reach the maxExecutions + // execute the sleep.php script until we reach maxExecutions newThread.onWork = func(thread *phpThread) bool { executionMutex.Lock() if executionCount >= maxExecutions { From a2f8d59dc6992c022027849e4cd4ee653b3729a0 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 2 Nov 2024 22:23:58 +0100 Subject: [PATCH 09/64] Cleanup. --- php_threads_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/php_threads_test.go b/php_threads_test.go index 3a074818a..2b0e25e34 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -24,9 +24,6 @@ func TestStartAndStopTheMainThread(t *testing.T) { } // We'll start 100 threads and check that their hooks work correctly -// onStartup => before the thread is ready -// onWork => while the thread is working -// onShutdown => after the thread is done func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { numThreads := 100 readyThreads := atomic.Uint64{} @@ -36,17 +33,23 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { for i := 0; i < numThreads; i++ { newThread := getInactivePHPThread() + + // onStartup => before the thread is ready newThread.onStartup = func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { readyThreads.Add(1) } } + + // onWork => while the thread is running (we stop here immediately) newThread.onWork = func(thread *phpThread) bool { if thread.threadIndex == newThread.threadIndex { workingThreads.Add(1) } return false // stop immediately } + + // onShutdown => after the thread is done newThread.onShutdown = func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { finishedThreads.Add(1) From 08254531d4f40da32e291a49619a4e75c3431a66 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sun, 3 Nov 2024 23:35:51 +0100 Subject: [PATCH 10/64] Adjusts watcher logic. --- frankenphp.go | 27 +++++++++++---------------- php_thread.go | 1 + php_threads_test.go | 2 ++ worker.go | 22 +++++++++++++--------- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/frankenphp.go b/frankenphp.go index 1db0714bc..b7fd24000 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -464,33 +464,28 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error fc.responseWriter = responseWriter fc.startedAt = time.Now() - isWorker := fc.responseWriter == nil isWorkerRequest := false rc := requestChan // Detect if a worker is available to handle this request - if !isWorker { - if worker, ok := workers[fc.scriptFilename]; ok { - isWorkerRequest = true - metrics.StartWorkerRequest(fc.scriptFilename) - rc = worker.requestChan - } else { - metrics.StartRequest() - } + if worker, ok := workers[fc.scriptFilename]; ok { + isWorkerRequest = true + metrics.StartWorkerRequest(fc.scriptFilename) + rc = worker.requestChan + } else { + metrics.StartRequest() } - + select { case <-done: case rc <- request: <-fc.done } - if !isWorker { - if isWorkerRequest { - metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) - } else { - metrics.StopRequest() - } + if isWorkerRequest { + metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) + } else { + metrics.StopRequest() } return nil diff --git a/php_thread.go b/php_thread.go index 351b7e356..0b7fdf12b 100644 --- a/php_thread.go +++ b/php_thread.go @@ -16,6 +16,7 @@ type phpThread struct { mainRequest *http.Request workerRequest *http.Request worker *worker + requestChan chan *http.Request threadIndex int // the index of the thread in the phpThreads slice isActive bool // whether the thread is currently running onStartup func(*phpThread) // the function to run when ready diff --git a/php_threads_test.go b/php_threads_test.go index 2b0e25e34..d91fde628 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -98,6 +98,8 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { } executionCount++ executionMutex.Unlock() + + // exit the loop and fail the test if the script fails if int(executeScriptCGI(scriptPath)) != 0 { return false } diff --git a/worker.go b/worker.go index 1ba3e110c..d77430b1d 100644 --- a/worker.go +++ b/worker.go @@ -79,6 +79,7 @@ func startNewWorkerThread(worker *worker) error { // onStartup => right before the thread is ready thread.onStartup = func(thread *phpThread) { thread.worker = worker + thread.requestChan = chan(*http.Request) metrics.ReadyWorker(worker.fileName) thread.backoff = newExponentialBackoff() } @@ -138,7 +139,6 @@ func restartWorkersOnFileChanges(workerOpts []workerOpt) error { } func restartWorkers(workerOpts []workerOpt) { - stopWorkers() workerShutdownWG.Wait() if err := initWorkers(workerOpts); err != nil { logger.Error("failed to restart workers when watching files") @@ -226,14 +226,20 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) } - if !executePHPFunction("opcache_reset") { - logger.Warn("opcache_reset failed") - } return C.bool(false) case r = <-thread.worker.requestChan: } + // a nil request is a signal for the worker to restart + if r == nil { + if !executePHPFunction("opcache_reset") { + logger.Warn("opcache_reset failed") + } + + return C.bool(false) + } + thread.workerRequest = r if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { @@ -256,23 +262,21 @@ func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] r := thread.getActiveRequest() fc := r.Context().Value(contextKey).(*FrankenPHPContext) - thread.workerRequest = nil maybeCloseContext(fc) + thread.workerRequest = nil + thread.Unpin() if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { c.Write(zap.String("worker", fc.scriptFilename), zap.String("url", r.RequestURI)) } - - thread.Unpin() } // when frankenphp_finish_request() is directly called from PHP // //export go_frankenphp_finish_request_manually func go_frankenphp_finish_request_manually(threadIndex C.uintptr_t) { - thread := phpThreads[threadIndex] - r := thread.getActiveRequest() + r := phpThreads[threadIndex].getActiveRequest() fc := r.Context().Value(contextKey).(*FrankenPHPContext) maybeCloseContext(fc) From 17d5cbe59f09538765c86474a297d5bf6eca4d08 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 4 Nov 2024 00:29:44 +0100 Subject: [PATCH 11/64] Adjusts the watcher logic. --- frankenphp.c | 44 ++++++++++++++++++---------------- frankenphp.go | 36 +++++++++------------------- frankenphp.h | 4 +--- php_thread.go | 9 +++---- php_threads.go | 2 +- php_threads_test.go | 4 ++-- worker.go | 57 +++++++++++++++++++++++++++++++-------------- 7 files changed, 83 insertions(+), 73 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 42bdfca39..3818ef44e 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -965,7 +965,26 @@ int frankenphp_request_startup() { return FAILURE; } -int frankenphp_execute_script(char *file_name) { +int frankenphp_execute_php_function(const char *php_function) { + zval retval = {0}; + zend_fcall_info fci = {0}; + zend_fcall_info_cache fci_cache = {0}; + zend_string *func_name = + zend_string_init(php_function, strlen(php_function), 0); + ZVAL_STR(&fci.function_name, func_name); + fci.size = sizeof fci; + fci.retval = &retval; + int success = 0; + + zend_try { success = zend_call_function(&fci, &fci_cache) == SUCCESS; } + zend_end_try(); + + zend_string_release(func_name); + + return success; +} + +int frankenphp_execute_script(char *file_name, bool clear_op_cache) { if (frankenphp_request_startup() == FAILURE) { free(file_name); file_name = NULL; @@ -1002,6 +1021,10 @@ int frankenphp_execute_script(char *file_name) { frankenphp_free_request_context(); frankenphp_request_shutdown(); + if (clear_op_cache) { + frankenphp_execute_php_function("opcache_reset"); + } + return status; } @@ -1160,22 +1183,3 @@ int frankenphp_execute_script_cli(char *script, int argc, char **argv) { return (intptr_t)exit_status; } - -int frankenphp_execute_php_function(const char *php_function) { - zval retval = {0}; - zend_fcall_info fci = {0}; - zend_fcall_info_cache fci_cache = {0}; - zend_string *func_name = - zend_string_init(php_function, strlen(php_function), 0); - ZVAL_STR(&fci.function_name, func_name); - fci.size = sizeof fci; - fci.retval = &retval; - int success = 0; - - zend_try { success = zend_call_function(&fci, &fci_cache) == SUCCESS; } - zend_end_try(); - - zend_string_release(func_name); - - return success; -} diff --git a/frankenphp.go b/frankenphp.go index b7fd24000..d7a279742 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -464,29 +464,24 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error fc.responseWriter = responseWriter fc.startedAt = time.Now() - isWorkerRequest := false - - rc := requestChan // Detect if a worker is available to handle this request if worker, ok := workers[fc.scriptFilename]; ok { - isWorkerRequest = true metrics.StartWorkerRequest(fc.scriptFilename) - rc = worker.requestChan - } else { - metrics.StartRequest() + worker.handleRequest(request) + <-fc.done + metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) + return nil } + + metrics.StartRequest() select { case <-done: - case rc <- request: + case requestChan <- request: <-fc.done } - if isWorkerRequest { - metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) - } else { - metrics.StopRequest() - } + metrics.StopRequest() return nil } @@ -583,7 +578,7 @@ func handleRequest(thread *phpThread) bool { panic(err) } - fc.exitStatus = executeScriptCGI(fc.scriptFilename) + fc.exitStatus = executeScriptCGI(fc.scriptFilename, false) return true } @@ -864,9 +859,9 @@ func go_log(message *C.char, level C.int) { } } -func executeScriptCGI(script string) C.int { +func executeScriptCGI(script string, clearOpCache bool) C.int { // scriptFilename is freed in frankenphp_execute_script() - exitStatus := C.frankenphp_execute_script(C.CString(script)) + exitStatus := C.frankenphp_execute_script(C.CString(script), C.bool(clearOpCache)) if exitStatus < 0 { panic(ScriptExecutionError) } @@ -899,12 +894,3 @@ func freeArgs(argv []*C.char) { C.free(unsafe.Pointer(arg)) } } - -func executePHPFunction(functionName string) bool { - cFunctionName := C.CString(functionName) - defer C.free(unsafe.Pointer(cFunctionName)) - - success := C.frankenphp_execute_php_function(cFunctionName) - - return success == 1 -} diff --git a/frankenphp.h b/frankenphp.h index 38d408fe6..e12be3dd7 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -50,13 +50,11 @@ int frankenphp_update_server_context( char *path_translated, char *request_uri, const char *content_type, char *auth_user, char *auth_password, int proto_num); int frankenphp_request_startup(); -int frankenphp_execute_script(char *file_name); +int frankenphp_execute_script(char *file_name, bool clear_opcache); void frankenphp_register_bulk_variables(go_string known_variables[27], php_variable *dynamic_variables, size_t size, zval *track_vars_array); int frankenphp_execute_script_cli(char *script, int argc, char **argv); -int frankenphp_execute_php_function(const char *php_function); - #endif diff --git a/php_thread.go b/php_thread.go index 0b7fdf12b..39eeb6cf3 100644 --- a/php_thread.go +++ b/php_thread.go @@ -7,6 +7,7 @@ import "C" import ( "fmt" "net/http" + "sync/atomic" "runtime" ) @@ -18,7 +19,7 @@ type phpThread struct { worker *worker requestChan chan *http.Request threadIndex int // the index of the thread in the phpThreads slice - isActive bool // whether the thread is currently running + isActive atomic.Bool // whether the thread is currently running onStartup func(*phpThread) // the function to run when ready onWork func(*phpThread) bool // the function to run in a loop when ready onShutdown func(*phpThread) // the function to run after shutdown @@ -34,7 +35,7 @@ func (thread phpThread) getActiveRequest() *http.Request { } func (thread *phpThread) run() error { - if thread.isActive { + if thread.isActive.Load() { return fmt.Errorf("thread is already running %d", thread.threadIndex) } if thread.onWork == nil { @@ -42,7 +43,7 @@ func (thread *phpThread) run() error { } threadsReadyWG.Add(1) shutdownWG.Add(1) - thread.isActive = true + thread.isActive.Store(true) if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { return fmt.Errorf("error creating thread %d", thread.threadIndex) } @@ -68,7 +69,7 @@ func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] - thread.isActive = false + thread.isActive.Store(false) thread.Unpin() if thread.onShutdown != nil { thread.onShutdown(thread) diff --git a/php_threads.go b/php_threads.go index 180594d66..405e1fb55 100644 --- a/php_threads.go +++ b/php_threads.go @@ -47,7 +47,7 @@ func startMainThread(numThreads int) error { func getInactivePHPThread() *phpThread { for _, thread := range phpThreads { - if !thread.isActive { + if !thread.isActive.Load() { return thread } } diff --git a/php_threads_test.go b/php_threads_test.go index d91fde628..b3df3b938 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -16,7 +16,7 @@ func TestStartAndStopTheMainThread(t *testing.T) { assert.Equal(t, 1, len(phpThreads)) assert.Equal(t, 0, phpThreads[0].threadIndex) - assert.False(t, phpThreads[0].isActive) + assert.False(t, phpThreads[0].isActive.Load()) assert.Nil(t, phpThreads[0].worker) drainPHPThreads() @@ -100,7 +100,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { executionMutex.Unlock() // exit the loop and fail the test if the script fails - if int(executeScriptCGI(scriptPath)) != 0 { + if int(executeScriptCGI(scriptPath, false)) != 0 { return false } diff --git a/worker.go b/worker.go index d77430b1d..0292ca919 100644 --- a/worker.go +++ b/worker.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sync" "sync/atomic" + "time" "github.com/dunglas/frankenphp/internal/watcher" "go.uber.org/zap" @@ -20,6 +21,8 @@ type worker struct { num int env PreparedEnv requestChan chan *http.Request + threads []*phpThread + threadMutex sync.RWMutex } var ( @@ -79,9 +82,12 @@ func startNewWorkerThread(worker *worker) error { // onStartup => right before the thread is ready thread.onStartup = func(thread *phpThread) { thread.worker = worker - thread.requestChan = chan(*http.Request) + thread.requestChan = make(chan *http.Request) metrics.ReadyWorker(worker.fileName) thread.backoff = newExponentialBackoff() + worker.threadMutex.Lock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() } // onWork => while the thread is working (in a loop) @@ -90,7 +96,8 @@ func startNewWorkerThread(worker *worker) error { return false } beforeWorkerScript(thread) - exitStatus := executeScriptCGI(thread.worker.fileName) + // TODO: opcache reset only if watcher is enabled + exitStatus := executeScriptCGI(thread.worker.fileName, true) afterWorkerScript(thread, exitStatus) return true @@ -119,6 +126,18 @@ func drainWorkers() { workers = make(map[string]*worker) } +// send a nil requests to workers to signal a restart +func restartWorkers() { + for _, worker := range workers { + worker.threadMutex.RLock() + for _, thread := range worker.threads { + thread.requestChan <- nil + } + worker.threadMutex.RUnlock() + } + time.Sleep(100 * time.Millisecond) // wait a bit before allowing another restart +} + func restartWorkersOnFileChanges(workerOpts []workerOpt) error { var directoriesToWatch []string for _, w := range workerOpts { @@ -128,9 +147,6 @@ func restartWorkersOnFileChanges(workerOpts []workerOpt) error { if !watcherIsEnabled { return nil } - restartWorkers := func() { - restartWorkers(workerOpts) - } if err := watcher.InitWatcher(directoriesToWatch, restartWorkers, getLogger()); err != nil { return err } @@ -138,15 +154,6 @@ func restartWorkersOnFileChanges(workerOpts []workerOpt) error { return nil } -func restartWorkers(workerOpts []workerOpt) { - workerShutdownWG.Wait() - if err := initWorkers(workerOpts); err != nil { - logger.Error("failed to restart workers when watching files") - panic(err) - } - logger.Info("workers restarted successfully") -} - func beforeWorkerScript(thread *phpThread) { worker := thread.worker @@ -212,6 +219,23 @@ func afterWorkerScript(thread *phpThread, exitStatus C.int) { }) } +func (worker *worker) handleRequest(r *http.Request) { + worker.threadMutex.RLock() + // dispatch requests to all worker threads in order + for _, thread := range worker.threads { + select { + case thread.requestChan <- r: + worker.threadMutex.RUnlock() + return + default: + } + } + worker.threadMutex.RUnlock() + // if no thread was available, fan the request out to all threads + // TODO: theoretically there could be autoscaling of threads here + worker.requestChan <- r +} + //export go_frankenphp_worker_handle_request_start func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { thread := phpThreads[threadIndex] @@ -228,15 +252,12 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { } return C.bool(false) + case r = <-thread.requestChan: case r = <-thread.worker.requestChan: } // a nil request is a signal for the worker to restart if r == nil { - if !executePHPFunction("opcache_reset") { - logger.Warn("opcache_reset failed") - } - return C.bool(false) } From 09e0ca677c14c3a20e90d4b38b52edb7199ac55b Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 4 Nov 2024 20:02:47 +0100 Subject: [PATCH 12/64] Fix opcache_reset race condition. --- frankenphp.c | 44 ++++++++++++-------------- frankenphp.go | 19 +++++++----- frankenphp.h | 3 +- php_threads_test.go | 2 +- worker.go | 76 +++++++++++++++++++++------------------------ 5 files changed, 70 insertions(+), 74 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 3818ef44e..42bdfca39 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -965,26 +965,7 @@ int frankenphp_request_startup() { return FAILURE; } -int frankenphp_execute_php_function(const char *php_function) { - zval retval = {0}; - zend_fcall_info fci = {0}; - zend_fcall_info_cache fci_cache = {0}; - zend_string *func_name = - zend_string_init(php_function, strlen(php_function), 0); - ZVAL_STR(&fci.function_name, func_name); - fci.size = sizeof fci; - fci.retval = &retval; - int success = 0; - - zend_try { success = zend_call_function(&fci, &fci_cache) == SUCCESS; } - zend_end_try(); - - zend_string_release(func_name); - - return success; -} - -int frankenphp_execute_script(char *file_name, bool clear_op_cache) { +int frankenphp_execute_script(char *file_name) { if (frankenphp_request_startup() == FAILURE) { free(file_name); file_name = NULL; @@ -1021,10 +1002,6 @@ int frankenphp_execute_script(char *file_name, bool clear_op_cache) { frankenphp_free_request_context(); frankenphp_request_shutdown(); - if (clear_op_cache) { - frankenphp_execute_php_function("opcache_reset"); - } - return status; } @@ -1183,3 +1160,22 @@ int frankenphp_execute_script_cli(char *script, int argc, char **argv) { return (intptr_t)exit_status; } + +int frankenphp_execute_php_function(const char *php_function) { + zval retval = {0}; + zend_fcall_info fci = {0}; + zend_fcall_info_cache fci_cache = {0}; + zend_string *func_name = + zend_string_init(php_function, strlen(php_function), 0); + ZVAL_STR(&fci.function_name, func_name); + fci.size = sizeof fci; + fci.retval = &retval; + int success = 0; + + zend_try { success = zend_call_function(&fci, &fci_cache) == SUCCESS; } + zend_end_try(); + + zend_string_release(func_name); + + return success; +} diff --git a/frankenphp.go b/frankenphp.go index d7a279742..1688a2eb4 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -349,10 +349,6 @@ func Init(options ...Option) error { // wait for all regular and worker threads to be ready for requests threadsReadyWG.Wait() - if err := restartWorkersOnFileChanges(opt.workers); err != nil { - return err - } - if c := logger.Check(zapcore.InfoLevel, "FrankenPHP started 🐘"); c != nil { c.Write(zap.String("php_version", Version().Version), zap.Int("num_threads", totalThreadCount)) } @@ -474,7 +470,7 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error } metrics.StartRequest() - + select { case <-done: case requestChan <- request: @@ -578,7 +574,7 @@ func handleRequest(thread *phpThread) bool { panic(err) } - fc.exitStatus = executeScriptCGI(fc.scriptFilename, false) + fc.exitStatus = executeScriptCGI(fc.scriptFilename) return true } @@ -859,9 +855,9 @@ func go_log(message *C.char, level C.int) { } } -func executeScriptCGI(script string, clearOpCache bool) C.int { +func executeScriptCGI(script string) C.int { // scriptFilename is freed in frankenphp_execute_script() - exitStatus := C.frankenphp_execute_script(C.CString(script), C.bool(clearOpCache)) + exitStatus := C.frankenphp_execute_script(C.CString(script)) if exitStatus < 0 { panic(ScriptExecutionError) } @@ -894,3 +890,10 @@ func freeArgs(argv []*C.char) { C.free(unsafe.Pointer(arg)) } } + +func executePHPFunction(functionName string) bool { + cFunctionName := C.CString(functionName) + defer C.free(unsafe.Pointer(cFunctionName)) + + return C.frankenphp_execute_php_function(cFunctionName) == 1 +} diff --git a/frankenphp.h b/frankenphp.h index e12be3dd7..b903148dc 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -50,11 +50,12 @@ int frankenphp_update_server_context( char *path_translated, char *request_uri, const char *content_type, char *auth_user, char *auth_password, int proto_num); int frankenphp_request_startup(); -int frankenphp_execute_script(char *file_name, bool clear_opcache); +int frankenphp_execute_script(char *file_name); void frankenphp_register_bulk_variables(go_string known_variables[27], php_variable *dynamic_variables, size_t size, zval *track_vars_array); int frankenphp_execute_script_cli(char *script, int argc, char **argv); +int frankenphp_execute_php_function(const char *php_function); #endif diff --git a/php_threads_test.go b/php_threads_test.go index b3df3b938..f80c6ba82 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -100,7 +100,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { executionMutex.Unlock() // exit the loop and fail the test if the script fails - if int(executeScriptCGI(scriptPath, false)) != 0 { + if int(executeScriptCGI(scriptPath)) != 0 { return false } diff --git a/worker.go b/worker.go index 0292ca919..df39edb7d 100644 --- a/worker.go +++ b/worker.go @@ -9,7 +9,6 @@ import ( "path/filepath" "sync" "sync/atomic" - "time" "github.com/dunglas/frankenphp/internal/watcher" "go.uber.org/zap" @@ -27,15 +26,20 @@ type worker struct { var ( watcherIsEnabled bool - workerShutdownWG sync.WaitGroup workersAreDone atomic.Bool workersDone chan interface{} - workers = make(map[string]*worker) + workers map[string]*worker + isRestarting atomic.Bool + workerRestartWG sync.WaitGroup + workerShutdownWG sync.WaitGroup ) func initWorkers(opt []workerOpt) error { + workers = make(map[string]*worker, len(opt)) workersDone = make(chan interface{}) workersAreDone.Store(false) + directoriesToWatch := getDirectoriesToWatch(opt) + watcherIsEnabled = len(directoriesToWatch) > 0 for _, o := range opt { worker, err := newWorker(o) @@ -49,6 +53,14 @@ func initWorkers(opt []workerOpt) error { } } + if len(directoriesToWatch) == 0 { + return nil + } + + if err := watcher.InitWatcher(directoriesToWatch, restartWorkers, getLogger()); err != nil { + return err + } + return nil } @@ -58,12 +70,6 @@ func newWorker(o workerOpt) (*worker, error) { return nil, fmt.Errorf("worker filename is invalid %q: %w", o.fileName, err) } - // if the worker already exists, return it - // it's necessary since we don't want to destroy the channels when restarting on file changes - if w, ok := workers[absFileName]; ok { - return w, nil - } - if o.env == nil { o.env = make(PreparedEnv, 1) } @@ -76,7 +82,6 @@ func newWorker(o workerOpt) (*worker, error) { } func startNewWorkerThread(worker *worker) error { - workerShutdownWG.Add(1) thread := getInactivePHPThread() // onStartup => right before the thread is ready @@ -86,8 +91,8 @@ func startNewWorkerThread(worker *worker) error { metrics.ReadyWorker(worker.fileName) thread.backoff = newExponentialBackoff() worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() } // onWork => while the thread is working (in a loop) @@ -95,9 +100,12 @@ func startNewWorkerThread(worker *worker) error { if workersAreDone.Load() { return false } + if watcherIsEnabled && isRestarting.Load() { + workerShutdownWG.Done() + workerRestartWG.Wait() + } beforeWorkerScript(thread) - // TODO: opcache reset only if watcher is enabled - exitStatus := executeScriptCGI(thread.worker.fileName, true) + exitStatus := executeScriptCGI(thread.worker.fileName) afterWorkerScript(thread, exitStatus) return true @@ -107,7 +115,6 @@ func startNewWorkerThread(worker *worker) error { thread.onShutdown = func(thread *phpThread) { thread.worker = nil thread.backoff = nil - workerShutdownWG.Done() } return thread.run() @@ -122,36 +129,27 @@ func drainWorkers() { watcher.DrainWatcher() watcherIsEnabled = false stopWorkers() - workerShutdownWG.Wait() - workers = make(map[string]*worker) } -// send a nil requests to workers to signal a restart func restartWorkers() { + workerRestartWG.Add(1) for _, worker := range workers { - worker.threadMutex.RLock() - for _, thread := range worker.threads { - thread.requestChan <- nil - } - worker.threadMutex.RUnlock() + workerShutdownWG.Add(worker.num) } - time.Sleep(100 * time.Millisecond) // wait a bit before allowing another restart + isRestarting.Store(true) + close(workersDone) + workerShutdownWG.Wait() + workersDone = make(chan interface{}) + isRestarting.Store(false) + workerRestartWG.Done() } -func restartWorkersOnFileChanges(workerOpts []workerOpt) error { - var directoriesToWatch []string +func getDirectoriesToWatch(workerOpts []workerOpt) []string { + directoriesToWatch := []string{} for _, w := range workerOpts { directoriesToWatch = append(directoriesToWatch, w.watch...) } - watcherIsEnabled = len(directoriesToWatch) > 0 - if !watcherIsEnabled { - return nil - } - if err := watcher.InitWatcher(directoriesToWatch, restartWorkers, getLogger()); err != nil { - return err - } - - return nil + return directoriesToWatch } func beforeWorkerScript(thread *phpThread) { @@ -250,17 +248,15 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) } + if isRestarting.Load() && !executePHPFunction("opcache_reset") { + logger.Error("failed to call opcache_reset") + } return C.bool(false) case r = <-thread.requestChan: case r = <-thread.worker.requestChan: } - // a nil request is a signal for the worker to restart - if r == nil { - return C.bool(false) - } - thread.workerRequest = r if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { From 7f13ada3e6a451f3310f3b24f99afb94dcfb4896 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 4 Nov 2024 20:33:37 +0100 Subject: [PATCH 13/64] Fixing merge conflicts and formatting. --- php_thread.go | 24 ++++++++++++------------ php_threads_test.go | 2 +- worker.go | 31 +++++++++++++++---------------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/php_thread.go b/php_thread.go index f5771209d..9692f2243 100644 --- a/php_thread.go +++ b/php_thread.go @@ -8,24 +8,24 @@ import "C" import ( "fmt" "net/http" - "sync/atomic" "runtime" + "sync/atomic" "unsafe" ) type phpThread struct { runtime.Pinner - mainRequest *http.Request - workerRequest *http.Request - worker *worker - requestChan chan *http.Request - threadIndex int // the index of the thread in the phpThreads slice - isActive atomic.Bool // whether the thread is currently running - onStartup func(*phpThread) // the function to run when ready - onWork func(*phpThread) bool // the function to run in a loop when ready - onShutdown func(*phpThread) // the function to run after shutdown - backoff *exponentialBackoff // backoff for worker failures + mainRequest *http.Request + workerRequest *http.Request + worker *worker + requestChan chan *http.Request + threadIndex int // the index of the thread in the phpThreads slice + isActive atomic.Bool // whether the thread is currently running + onStartup func(*phpThread) // the function to run when ready + onWork func(*phpThread) bool // the function to run in a loop when ready + onShutdown func(*phpThread) // the function to run after shutdown + backoff *exponentialBackoff // backoff for worker failures knownVariableKeys map[string]*C.zend_string } @@ -64,7 +64,7 @@ func (thread *phpThread) pinString(s string) *C.char { // C strings must be null-terminated func (thread *phpThread) pinCString(s string) *C.char { - return thread.pinString(s+"\x00") + return thread.pinString(s + "\x00") } //export go_frankenphp_on_thread_startup diff --git a/php_threads_test.go b/php_threads_test.go index f80c6ba82..c33932f7d 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -85,7 +85,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { newThread.onStartup = func(thread *phpThread) { r, _ := http.NewRequest(http.MethodGet, "sleep.php", nil) r, _ = NewRequestWithContext(r, WithRequestDocumentRoot("/", false)) - assert.NoError(t, updateServerContext(r, true, false)) + assert.NoError(t, updateServerContext(thread, r, true, false)) thread.mainRequest = r } diff --git a/worker.go b/worker.go index f74fd3eee..360cd4953 100644 --- a/worker.go +++ b/worker.go @@ -25,13 +25,13 @@ type worker struct { } var ( - watcherIsEnabled bool - workersAreDone atomic.Bool - workersDone chan interface{} - workers map[string]*worker - isRestarting atomic.Bool - workerRestartWG sync.WaitGroup - workerShutdownWG sync.WaitGroup + workers map[string]*worker + workersDone chan interface{} + watcherIsEnabled bool + workersAreDone atomic.Bool + workersAreRestarting atomic.Bool + workerRestartWG sync.WaitGroup + workerShutdownWG sync.WaitGroup ) func initWorkers(opt []workerOpt) error { @@ -101,7 +101,7 @@ func startNewWorkerThread(worker *worker) error { if workersAreDone.Load() { return false } - if watcherIsEnabled && isRestarting.Load() { + if watcherIsEnabled && workersAreRestarting.Load() { workerShutdownWG.Done() workerRestartWG.Wait() } @@ -134,15 +134,15 @@ func drainWorkers() { func restartWorkers() { workerRestartWG.Add(1) + defer workerRestartWG.Done() for _, worker := range workers { workerShutdownWG.Add(worker.num) } - isRestarting.Store(true) + workersAreRestarting.Store(true) close(workersDone) workerShutdownWG.Wait() workersDone = make(chan interface{}) - isRestarting.Store(false) - workerRestartWG.Done() + workersAreRestarting.Store(false) } func getDirectoriesToWatch(workerOpts []workerOpt) []string { @@ -175,7 +175,7 @@ func beforeWorkerScript(thread *phpThread) { panic(err) } - if err := updateServerContext(r, true, false); err != nil { + if err := updateServerContext(thread, r, true, false); err != nil { panic(err) } @@ -249,14 +249,15 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) } - if isRestarting.Load() && !executePHPFunction("opcache_reset") { + + // execute opcache_reset if the restart was triggered by the watcher + if watcherIsEnabled && workersAreRestarting.Load() && !executePHPFunction("opcache_reset") { logger.Error("failed to call opcache_reset") } return C.bool(false) case r = <-thread.requestChan: case r = <-thread.worker.requestChan: - case r = <-thread.requestChan: } thread.workerRequest = r @@ -307,6 +308,4 @@ func go_frankenphp_finish_request_manually(threadIndex C.uintptr_t) { if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { c.Write(zap.String("url", r.RequestURI)) } - - thread.Unpin() } From 13fb4bb729d143f625638320e877f6d0433143bf Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 11:39:51 +0100 Subject: [PATCH 14/64] Prevents overlapping of TSRM reservation and script execution. --- frankenphp.c | 6 +- frankenphp.go | 16 ++--- php_thread.go | 81 ++++++++++++++--------- php_threads.go | 24 ++++++- php_threads_test.go | 153 +++++++++++++++++++++++++++++--------------- worker.go | 73 +++++++++------------ 6 files changed, 210 insertions(+), 143 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 7403cabb3..79bcfb989 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -822,8 +822,6 @@ static void *php_thread(void *arg) { cfg_get_string("filter.default", &default_filter); should_filter_var = default_filter != NULL; - go_frankenphp_on_thread_startup(thread_index); - // perform work until go signals to stop while (go_frankenphp_on_thread_work(thread_index)) { } @@ -853,13 +851,11 @@ static void *php_main(void *arg) { exit(EXIT_FAILURE); } - intptr_t num_threads = (intptr_t)arg; - set_thread_name("php-main"); #ifdef ZTS #if (PHP_VERSION_ID >= 80300) - php_tsrm_startup_ex(num_threads); + php_tsrm_startup_ex((intptr_t)arg); #else php_tsrm_startup(); #endif diff --git a/frankenphp.go b/frankenphp.go index a405d0af3..a9cab8ced 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -336,19 +336,13 @@ func Init(options ...Option) error { for i := 0; i < totalThreadCount-workerThreadCount; i++ { thread := getInactivePHPThread() - thread.onWork = handleRequest - if err := thread.run(); err != nil { - return err - } + thread.setHooks(nil, handleRequest, nil) } if err := initWorkers(opt.workers); err != nil { return err } - // wait for all regular and worker threads to be ready for requests - threadsReadyWG.Wait() - if c := logger.Check(zapcore.InfoLevel, "FrankenPHP started 🐘"); c != nil { c.Write(zap.String("php_version", Version().Version), zap.Int("num_threads", totalThreadCount)) } @@ -556,10 +550,10 @@ func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string return true, value // Return 1 to indicate success } -func handleRequest(thread *phpThread) bool { +func handleRequest(thread *phpThread) { select { case <-done: - return false + return case r := <-requestChan: thread.mainRequest = r @@ -576,12 +570,10 @@ func handleRequest(thread *phpThread) bool { if err := updateServerContext(thread, r, true, false); err != nil { rejectRequest(fc.responseWriter, err.Error()) - return true + return } fc.exitStatus = executeScriptCGI(fc.scriptFilename) - - return true } } diff --git a/php_thread.go b/php_thread.go index 9692f2243..d07cf4c1a 100644 --- a/php_thread.go +++ b/php_thread.go @@ -6,7 +6,6 @@ package frankenphp // #include "frankenphp.h" import "C" import ( - "fmt" "net/http" "runtime" "sync/atomic" @@ -20,12 +19,14 @@ type phpThread struct { workerRequest *http.Request worker *worker requestChan chan *http.Request - threadIndex int // the index of the thread in the phpThreads slice - isActive atomic.Bool // whether the thread is currently running - onStartup func(*phpThread) // the function to run when ready - onWork func(*phpThread) bool // the function to run in a loop when ready - onShutdown func(*phpThread) // the function to run after shutdown - backoff *exponentialBackoff // backoff for worker failures + done chan struct{} // to signal the thread to stop the + threadIndex int // the index of the thread in the phpThreads slice + isActive atomic.Bool // whether the thread is currently running + isReady atomic.Bool // whether the thread is ready for work + onStartup func(*phpThread) // the function to run when ready + onWork func(*phpThread) // the function to run in a loop when ready + onShutdown func(*phpThread) // the function to run after shutdown + backoff *exponentialBackoff // backoff for worker failures knownVariableKeys map[string]*C.zend_string } @@ -37,21 +38,36 @@ func (thread phpThread) getActiveRequest() *http.Request { return thread.mainRequest } -func (thread *phpThread) run() error { - if thread.isActive.Load() { - return fmt.Errorf("thread is already running %d", thread.threadIndex) - } - if thread.onWork == nil { - return fmt.Errorf("thread.onWork must be defined %d", thread.threadIndex) +func (thread *phpThread) setInactive() { + thread.isActive.Store(false) + thread.onWork = func(thread *phpThread) { + thread.requestChan = make(chan *http.Request) + select { + case <-done: + case <-thread.done: + } } - threadsReadyWG.Add(1) - shutdownWG.Add(1) +} + +func (thread *phpThread) setHooks(onStartup func(*phpThread), onWork func(*phpThread), onShutdown func(*phpThread)) { thread.isActive.Store(true) - if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("error creating thread %d", thread.threadIndex) + + // to avoid race conditions, the thread sets its own hooks on startup + thread.onStartup = func(thread *phpThread) { + if thread.onShutdown != nil { + thread.onShutdown(thread) + } + thread.onStartup = onStartup + thread.onWork = onWork + thread.onShutdown = onShutdown + if thread.onStartup != nil { + thread.onStartup(thread) + } } - return nil + threadsReadyWG.Add(1) + close(thread.done) + thread.isReady.Store(false) } // Pin a string that is not null-terminated @@ -67,25 +83,32 @@ func (thread *phpThread) pinCString(s string) *C.char { return thread.pinString(s + "\x00") } -//export go_frankenphp_on_thread_startup -func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { - thread := phpThreads[threadIndex] - if thread.onStartup != nil { - thread.onStartup(thread) - } - threadsReadyWG.Done() -} - //export go_frankenphp_on_thread_work func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { + // first check if FrankPHP is shutting down + if threadsAreDone.Load() { + return C.bool(false) + } thread := phpThreads[threadIndex] - return C.bool(thread.onWork(thread)) + + // if the thread is not ready, set it up + if !thread.isReady.Load() { + thread.isReady.Store(true) + thread.done = make(chan struct{}) + if thread.onStartup != nil { + thread.onStartup(thread) + } + threadsReadyWG.Done() + } + + // do the actual work + thread.onWork(thread) + return C.bool(true) } //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { thread := phpThreads[threadIndex] - thread.isActive.Store(false) thread.Unpin() if thread.onShutdown != nil { thread.onShutdown(thread) diff --git a/php_threads.go b/php_threads.go index 405e1fb55..76f23b173 100644 --- a/php_threads.go +++ b/php_threads.go @@ -4,7 +4,9 @@ package frankenphp // #include "frankenphp.h" import "C" import ( + "fmt" "sync" + "sync/atomic" ) var ( @@ -14,19 +16,39 @@ var ( threadsReadyWG sync.WaitGroup shutdownWG sync.WaitGroup done chan struct{} + threadsAreDone atomic.Bool ) // reserve a fixed number of PHP threads on the go side func initPHPThreads(numThreads int) error { + threadsAreDone.Store(false) done = make(chan struct{}) phpThreads = make([]*phpThread, numThreads) for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{threadIndex: i} } - return startMainThread(numThreads) + logger.Warn("initializing main thread") + if err := startMainThread(numThreads); err != nil { + return err + } + + // initialize all threads as inactive + threadsReadyWG.Add(len(phpThreads)) + shutdownWG.Add(len(phpThreads)) + for _, thread := range phpThreads { + logger.Warn("initializing thread") + thread.setInactive() + logger.Warn("thread initialized") + if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { + return fmt.Errorf("unable to create thread %d", thread.threadIndex) + } + } + threadsReadyWG.Wait() + return nil } func drainPHPThreads() { + threadsAreDone.Store(true) close(done) shutdownWG.Wait() mainThreadShutdownWG.Done() diff --git a/php_threads_test.go b/php_threads_test.go index c33932f7d..f745b427b 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -12,7 +12,8 @@ import ( ) func TestStartAndStopTheMainThread(t *testing.T) { - initPHPThreads(1) // reserve 1 thread + logger = zap.NewNop() // the logger needs to not be nil + initPHPThreads(1) // reserve 1 thread assert.Equal(t, 1, len(phpThreads)) assert.Equal(t, 0, phpThreads[0].threadIndex) @@ -25,45 +26,45 @@ func TestStartAndStopTheMainThread(t *testing.T) { // We'll start 100 threads and check that their hooks work correctly func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { + logger = zap.NewNop() // the logger needs to not be nil numThreads := 100 readyThreads := atomic.Uint64{} finishedThreads := atomic.Uint64{} workingThreads := atomic.Uint64{} initPHPThreads(numThreads) + workWG := sync.WaitGroup{} + workWG.Add(numThreads) for i := 0; i < numThreads; i++ { newThread := getInactivePHPThread() - - // onStartup => before the thread is ready - newThread.onStartup = func(thread *phpThread) { - if thread.threadIndex == newThread.threadIndex { - readyThreads.Add(1) - } - } - - // onWork => while the thread is running (we stop here immediately) - newThread.onWork = func(thread *phpThread) bool { - if thread.threadIndex == newThread.threadIndex { - workingThreads.Add(1) - } - return false // stop immediately - } - - // onShutdown => after the thread is done - newThread.onShutdown = func(thread *phpThread) { - if thread.threadIndex == newThread.threadIndex { - finishedThreads.Add(1) - } - } - newThread.run() + newThread.setHooks( + // onStartup => before the thread is ready + func(thread *phpThread) { + if thread.threadIndex == newThread.threadIndex { + readyThreads.Add(1) + } + }, + // onWork => while the thread is running (we stop here immediately) + func(thread *phpThread) { + if thread.threadIndex == newThread.threadIndex { + workingThreads.Add(1) + } + workWG.Done() + newThread.setInactive() + }, + // onShutdown => after the thread is done + func(thread *phpThread) { + if thread.threadIndex == newThread.threadIndex { + finishedThreads.Add(1) + } + }, + ) } - threadsReadyWG.Wait() - - assert.Equal(t, numThreads, int(readyThreads.Load())) - + workWG.Wait() drainPHPThreads() + assert.Equal(t, numThreads, int(readyThreads.Load())) assert.Equal(t, numThreads, int(workingThreads.Load())) assert.Equal(t, numThreads, int(finishedThreads.Load())) } @@ -77,39 +78,87 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { executionCount := 0 scriptPath, _ := filepath.Abs("./testdata/sleep.php") initPHPThreads(numThreads) + workWG := sync.WaitGroup{} + workWG.Add(maxExecutions) for i := 0; i < numThreads; i++ { - newThread := getInactivePHPThread() + getInactivePHPThread().setHooks( + // onStartup => fake a request on startup (like a worker would do) + func(thread *phpThread) { + r, _ := http.NewRequest(http.MethodGet, "sleep.php", nil) + r, _ = NewRequestWithContext(r, WithRequestDocumentRoot("/", false)) + assert.NoError(t, updateServerContext(thread, r, true, false)) + thread.mainRequest = r + }, + // onWork => execute the sleep.php script until we reach maxExecutions + func(thread *phpThread) { + executionMutex.Lock() + if executionCount >= maxExecutions { + executionMutex.Unlock() + thread.setInactive() + return + } + executionCount++ + workWG.Done() + executionMutex.Unlock() - // fake a request on startup (like a worker would do) - newThread.onStartup = func(thread *phpThread) { - r, _ := http.NewRequest(http.MethodGet, "sleep.php", nil) - r, _ = NewRequestWithContext(r, WithRequestDocumentRoot("/", false)) - assert.NoError(t, updateServerContext(thread, r, true, false)) - thread.mainRequest = r - } + // exit the loop and fail the test if the script fails + if int(executeScriptCGI(scriptPath)) != 0 { + panic("script execution failed: " + scriptPath) + } + }, + // onShutdown => nothing to do here + nil, + ) + } - // execute the sleep.php script until we reach maxExecutions - newThread.onWork = func(thread *phpThread) bool { - executionMutex.Lock() - if executionCount >= maxExecutions { - executionMutex.Unlock() - return false - } - executionCount++ - executionMutex.Unlock() + workWG.Wait() + drainPHPThreads() - // exit the loop and fail the test if the script fails - if int(executeScriptCGI(scriptPath)) != 0 { - return false - } + assert.Equal(t, maxExecutions, executionCount) +} - return true +func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { + logger = zap.NewNop() // the logger needs to not be nil + numThreads := 100 + numConversions := 10 + startUpTypes := make([]atomic.Uint64, numConversions) + workTypes := make([]atomic.Uint64, numConversions) + shutdownTypes := make([]atomic.Uint64, numConversions) + workWG := sync.WaitGroup{} + + initPHPThreads(numThreads) + + for i := 0; i < numConversions; i++ { + workWG.Add(numThreads) + numberOfConversion := i + for j := 0; j < numThreads; j++ { + getInactivePHPThread().setHooks( + // onStartup => before the thread is ready + func(thread *phpThread) { + startUpTypes[numberOfConversion].Add(1) + }, + // onWork => while the thread is running + func(thread *phpThread) { + workTypes[numberOfConversion].Add(1) + thread.setInactive() + workWG.Done() + }, + // onShutdown => after the thread is done + func(thread *phpThread) { + shutdownTypes[numberOfConversion].Add(1) + }, + ) } - newThread.run() + workWG.Wait() } drainPHPThreads() - assert.Equal(t, maxExecutions, executionCount) + // each type of thread needs to have started, worked and stopped the same amount of times + for i := 0; i < numConversions; i++ { + assert.Equal(t, numThreads, int(startUpTypes[i].Load())) + assert.Equal(t, numThreads, int(workTypes[i].Load())) + assert.Equal(t, numThreads, int(shutdownTypes[i].Load())) + } } diff --git a/worker.go b/worker.go index 360cd4953..953b77f3c 100644 --- a/worker.go +++ b/worker.go @@ -28,7 +28,6 @@ var ( workers map[string]*worker workersDone chan interface{} watcherIsEnabled bool - workersAreDone atomic.Bool workersAreRestarting atomic.Bool workerRestartWG sync.WaitGroup workerShutdownWG sync.WaitGroup @@ -37,7 +36,6 @@ var ( func initWorkers(opt []workerOpt) error { workers = make(map[string]*worker, len(opt)) workersDone = make(chan interface{}) - workersAreDone.Store(false) directoriesToWatch := getDirectoriesToWatch(opt) watcherIsEnabled = len(directoriesToWatch) > 0 @@ -48,9 +46,7 @@ func initWorkers(opt []workerOpt) error { return err } for i := 0; i < worker.num; i++ { - if err := startNewWorkerThread(worker); err != nil { - return err - } + worker.startNewThread() } } @@ -82,53 +78,42 @@ func newWorker(o workerOpt) (*worker, error) { return w, nil } -func startNewWorkerThread(worker *worker) error { - thread := getInactivePHPThread() - - // onStartup => right before the thread is ready - thread.onStartup = func(thread *phpThread) { - thread.worker = worker - thread.requestChan = make(chan *http.Request) - metrics.ReadyWorker(worker.fileName) - thread.backoff = newExponentialBackoff() - worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() - } - - // onWork => while the thread is working (in a loop) - thread.onWork = func(thread *phpThread) bool { - if workersAreDone.Load() { - return false - } - if watcherIsEnabled && workersAreRestarting.Load() { - workerShutdownWG.Done() - workerRestartWG.Wait() - } - beforeWorkerScript(thread) - exitStatus := executeScriptCGI(thread.worker.fileName) - afterWorkerScript(thread, exitStatus) - - return true - } - - // onShutdown => after the thread is done - thread.onShutdown = func(thread *phpThread) { - thread.worker = nil - thread.backoff = nil - } - - return thread.run() +func (worker *worker) startNewThread() { + getInactivePHPThread().setHooks( + // onStartup => right before the thread is ready + func(thread *phpThread) { + thread.worker = worker + thread.requestChan = make(chan *http.Request) + metrics.ReadyWorker(worker.fileName) + thread.backoff = newExponentialBackoff() + worker.threadMutex.Lock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() + }, + // onWork => while the thread is working (in a loop) + func(thread *phpThread) { + if watcherIsEnabled && workersAreRestarting.Load() { + workerShutdownWG.Done() + workerRestartWG.Wait() + } + beforeWorkerScript(thread) + exitStatus := executeScriptCGI(thread.worker.fileName) + afterWorkerScript(thread, exitStatus) + }, + // onShutdown => after the thread is done + func(thread *phpThread) { + thread.worker = nil + thread.backoff = nil + }, + ) } func stopWorkers() { - workersAreDone.Store(true) close(workersDone) } func drainWorkers() { watcher.DrainWatcher() - watcherIsEnabled = false stopWorkers() } From a8a00c83724281f687e6a6cb63e13714bcd802b9 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 13:07:36 +0100 Subject: [PATCH 15/64] Adjustments as suggested by @dunglas. --- frankenphp.c | 8 ++++---- frankenphp.go | 4 ++-- frankenphp.h | 2 +- php_threads.go | 4 ++-- php_threads_test.go | 6 +++--- testdata/sleep.php | 2 +- worker.go | 6 +++--- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 79bcfb989..7a357e093 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -243,7 +243,7 @@ PHP_FUNCTION(frankenphp_finish_request) { /* {{{ */ php_header(); if (ctx->has_active_request) { - go_frankenphp_finish_request_manually(thread_index); + go_frankenphp_finish_php_request(thread_index); } ctx->finished = true; @@ -913,13 +913,13 @@ int frankenphp_new_main_thread(int num_threads) { return pthread_detach(thread); } -int frankenphp_new_php_thread(uintptr_t thread_index) { +bool frankenphp_new_php_thread(uintptr_t thread_index) { pthread_t thread; if (pthread_create(&thread, NULL, &php_thread, (void *)thread_index) != 0) { - return 1; + return false; } pthread_detach(thread); - return 0; + return true; } int frankenphp_request_startup() { diff --git a/frankenphp.go b/frankenphp.go index a9cab8ced..ebaf95584 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -573,7 +573,7 @@ func handleRequest(thread *phpThread) { return } - fc.exitStatus = executeScriptCGI(fc.scriptFilename) + fc.exitStatus = executeScriptClassic(fc.scriptFilename) } } @@ -787,7 +787,7 @@ func go_log(message *C.char, level C.int) { } } -func executeScriptCGI(script string) C.int { +func executeScriptClassic(script string) C.int { // scriptFilename is freed in frankenphp_execute_script() exitStatus := C.frankenphp_execute_script(C.CString(script)) if exitStatus < 0 { diff --git a/frankenphp.h b/frankenphp.h index ca91fc2d4..6d2e4efe2 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -41,7 +41,7 @@ typedef struct frankenphp_config { frankenphp_config frankenphp_get_config(); int frankenphp_new_main_thread(int num_threads); -int frankenphp_new_php_thread(uintptr_t thread_index); +bool frankenphp_new_php_thread(uintptr_t thread_index); int frankenphp_update_server_context( bool create, bool has_main_request, bool has_active_request, diff --git a/php_threads.go b/php_threads.go index 76f23b173..07da30db0 100644 --- a/php_threads.go +++ b/php_threads.go @@ -39,8 +39,8 @@ func initPHPThreads(numThreads int) error { logger.Warn("initializing thread") thread.setInactive() logger.Warn("thread initialized") - if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) != 0 { - return fmt.Errorf("unable to create thread %d", thread.threadIndex) + if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { + panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) } } threadsReadyWG.Wait() diff --git a/php_threads_test.go b/php_threads_test.go index f745b427b..627947dee 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -11,11 +11,11 @@ import ( "go.uber.org/zap" ) -func TestStartAndStopTheMainThread(t *testing.T) { +func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { logger = zap.NewNop() // the logger needs to not be nil initPHPThreads(1) // reserve 1 thread - assert.Equal(t, 1, len(phpThreads)) + assert.Len(t, phpThreads, 1) assert.Equal(t, 0, phpThreads[0].threadIndex) assert.False(t, phpThreads[0].isActive.Load()) assert.Nil(t, phpThreads[0].worker) @@ -103,7 +103,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { executionMutex.Unlock() // exit the loop and fail the test if the script fails - if int(executeScriptCGI(scriptPath)) != 0 { + if int(executeScriptClassic(scriptPath)) != 0 { panic("script execution failed: " + scriptPath) } }, diff --git a/testdata/sleep.php b/testdata/sleep.php index 1b1a66d02..d2c78b865 100644 --- a/testdata/sleep.php +++ b/testdata/sleep.php @@ -1,4 +1,4 @@ after the thread is done @@ -284,8 +284,8 @@ func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { // when frankenphp_finish_request() is directly called from PHP // -//export go_frankenphp_finish_request_manually -func go_frankenphp_finish_request_manually(threadIndex C.uintptr_t) { +//export go_frankenphp_finish_php_request +func go_frankenphp_finish_php_request(threadIndex C.uintptr_t) { r := phpThreads[threadIndex].getActiveRequest() fc := r.Context().Value(contextKey).(*FrankenPHPContext) maybeCloseContext(fc) From b4dd1382a7358cd793c508c6b6a114a0df7518a4 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 13:31:27 +0100 Subject: [PATCH 16/64] Adds error assertions. --- php_threads_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/php_threads_test.go b/php_threads_test.go index 627947dee..ea31ef287 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -13,7 +13,7 @@ import ( func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { logger = zap.NewNop() // the logger needs to not be nil - initPHPThreads(1) // reserve 1 thread + assert.NoError(t, initPHPThreads(1)) // reserve 1 thread assert.Len(t, phpThreads, 1) assert.Equal(t, 0, phpThreads[0].threadIndex) @@ -31,10 +31,11 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { readyThreads := atomic.Uint64{} finishedThreads := atomic.Uint64{} workingThreads := atomic.Uint64{} - initPHPThreads(numThreads) workWG := sync.WaitGroup{} workWG.Add(numThreads) + assert.NoError(t, initPHPThreads(numThreads)) + for i := 0; i < numThreads; i++ { newThread := getInactivePHPThread() newThread.setHooks( @@ -77,10 +78,11 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { executionMutex := sync.Mutex{} executionCount := 0 scriptPath, _ := filepath.Abs("./testdata/sleep.php") - initPHPThreads(numThreads) workWG := sync.WaitGroup{} workWG.Add(maxExecutions) + assert.NoError(t, initPHPThreads(numThreads)) + for i := 0; i < numThreads; i++ { getInactivePHPThread().setHooks( // onStartup => fake a request on startup (like a worker would do) @@ -118,6 +120,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { assert.Equal(t, maxExecutions, executionCount) } +// TODO: Make this test more chaotic func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { logger = zap.NewNop() // the logger needs to not be nil numThreads := 100 @@ -127,7 +130,7 @@ func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { shutdownTypes := make([]atomic.Uint64, numConversions) workWG := sync.WaitGroup{} - initPHPThreads(numThreads) + assert.NoError(t, initPHPThreads(numThreads)) for i := 0; i < numConversions; i++ { workWG.Add(numThreads) From 03f98fadb09c9585a27ee5da54905467bb531fb1 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 14:41:43 +0100 Subject: [PATCH 17/64] Adds comments. --- php_thread.go | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/php_thread.go b/php_thread.go index d07cf4c1a..8e9232c78 100644 --- a/php_thread.go +++ b/php_thread.go @@ -15,18 +15,28 @@ import ( type phpThread struct { runtime.Pinner - mainRequest *http.Request - workerRequest *http.Request - worker *worker - requestChan chan *http.Request - done chan struct{} // to signal the thread to stop the - threadIndex int // the index of the thread in the phpThreads slice - isActive atomic.Bool // whether the thread is currently running - isReady atomic.Bool // whether the thread is ready for work - onStartup func(*phpThread) // the function to run when ready - onWork func(*phpThread) // the function to run in a loop when ready - onShutdown func(*phpThread) // the function to run after shutdown - backoff *exponentialBackoff // backoff for worker failures + mainRequest *http.Request + workerRequest *http.Request + requestChan chan *http.Request + worker *worker + + // the index in the phpThreads slice + threadIndex int + // whether the thread has work assigned to it + isActive atomic.Bool + // whether the thread is ready for work + isReady atomic.Bool + // right before the first work iteration + onStartup func(*phpThread) + // the actual work iteration (done in a loop) + onWork func(*phpThread) + // after the thread is done + onShutdown func(*phpThread) + // chan to signal the thread to stop the current work iteration + done chan struct{} + // exponential backoff for worker failures + backoff *exponentialBackoff + // known $_SERVER key names knownVariableKeys map[string]*C.zend_string } @@ -38,6 +48,7 @@ func (thread phpThread) getActiveRequest() *http.Request { return thread.mainRequest } +// TODO: Also consider this case: work => inactive => work func (thread *phpThread) setInactive() { thread.isActive.Store(false) thread.onWork = func(thread *phpThread) { @@ -65,6 +76,7 @@ func (thread *phpThread) setHooks(onStartup func(*phpThread), onWork func(*phpTh } } + // we signal to the thread to stop it's current execution and call the onStartup hook threadsReadyWG.Add(1) close(thread.done) thread.isReady.Store(false) From e52dd0fedb9875a41dcd3b28be5d575c5d0bd78a Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 14:46:57 +0100 Subject: [PATCH 18/64] Removes logs and explicitly compares to C.false. --- php_threads.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/php_threads.go b/php_threads.go index 07da30db0..63b96c4d3 100644 --- a/php_threads.go +++ b/php_threads.go @@ -27,7 +27,6 @@ func initPHPThreads(numThreads int) error { for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{threadIndex: i} } - logger.Warn("initializing main thread") if err := startMainThread(numThreads); err != nil { return err } @@ -36,10 +35,8 @@ func initPHPThreads(numThreads int) error { threadsReadyWG.Add(len(phpThreads)) shutdownWG.Add(len(phpThreads)) for _, thread := range phpThreads { - logger.Warn("initializing thread") thread.setInactive() - logger.Warn("thread initialized") - if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { + if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) == C.false { panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) } } From cd98e33e973a23cbd658888149b0ecd6af0df03a Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 14:49:10 +0100 Subject: [PATCH 19/64] Resets check. --- php_threads.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php_threads.go b/php_threads.go index 63b96c4d3..6282b19f2 100644 --- a/php_threads.go +++ b/php_threads.go @@ -36,7 +36,7 @@ func initPHPThreads(numThreads int) error { shutdownWG.Add(len(phpThreads)) for _, thread := range phpThreads { thread.setInactive() - if C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) == C.false { + if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) } } From 4e2a2c61a294580c4e79b18d79feb5e19fd6ef72 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 14:52:02 +0100 Subject: [PATCH 20/64] Adds cast for safety. --- php_threads.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/php_threads.go b/php_threads.go index 6282b19f2..6233adf1c 100644 --- a/php_threads.go +++ b/php_threads.go @@ -36,7 +36,7 @@ func initPHPThreads(numThreads int) error { shutdownWG.Add(len(phpThreads)) for _, thread := range phpThreads { thread.setInactive() - if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { + if !bool(C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex))) { panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) } } From c51eb931949484903198b4c0a6e052e97ccae8f5 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 5 Nov 2024 20:33:03 +0100 Subject: [PATCH 21/64] Fixes waitgroup overflow. --- php_thread.go | 2 +- php_threads.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/php_thread.go b/php_thread.go index 8e9232c78..259eca587 100644 --- a/php_thread.go +++ b/php_thread.go @@ -76,7 +76,7 @@ func (thread *phpThread) setHooks(onStartup func(*phpThread), onWork func(*phpTh } } - // we signal to the thread to stop it's current execution and call the onStartup hook + // signal to the thread to stop it's current execution and call the onStartup hook threadsReadyWG.Add(1) close(thread.done) thread.isReady.Store(false) diff --git a/php_threads.go b/php_threads.go index 6233adf1c..edc2bbfda 100644 --- a/php_threads.go +++ b/php_threads.go @@ -21,6 +21,7 @@ var ( // reserve a fixed number of PHP threads on the go side func initPHPThreads(numThreads int) error { + threadsReadyWG = sync.WaitGroup{} threadsAreDone.Store(false) done = make(chan struct{}) phpThreads = make([]*phpThread, numThreads) @@ -36,7 +37,7 @@ func initPHPThreads(numThreads int) error { shutdownWG.Add(len(phpThreads)) for _, thread := range phpThreads { thread.setInactive() - if !bool(C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex))) { + if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) } } From 89d8e267d8df3664c7c01ff2af9fddbd3a74b9de Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Wed, 6 Nov 2024 13:45:13 +0100 Subject: [PATCH 22/64] Resolves waitgroup race condition on startup. --- frankenphp.go | 3 +-- php_thread.go | 10 ++++++---- php_threads.go | 5 +++++ php_threads_test.go | 6 +++--- worker.go | 4 ++-- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/frankenphp.go b/frankenphp.go index ebaf95584..15870a9e4 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -335,8 +335,7 @@ func Init(options ...Option) error { } for i := 0; i < totalThreadCount-workerThreadCount; i++ { - thread := getInactivePHPThread() - thread.setHooks(nil, handleRequest, nil) + getInactivePHPThread().setActive(nil, handleRequest, nil) } if err := initWorkers(opt.workers); err != nil { diff --git a/php_thread.go b/php_thread.go index 259eca587..183a31ca6 100644 --- a/php_thread.go +++ b/php_thread.go @@ -40,7 +40,7 @@ type phpThread struct { knownVariableKeys map[string]*C.zend_string } -func (thread phpThread) getActiveRequest() *http.Request { +func (thread *phpThread) getActiveRequest() *http.Request { if thread.workerRequest != nil { return thread.workerRequest } @@ -60,7 +60,7 @@ func (thread *phpThread) setInactive() { } } -func (thread *phpThread) setHooks(onStartup func(*phpThread), onWork func(*phpThread), onShutdown func(*phpThread)) { +func (thread *phpThread) setActive(onStartup func(*phpThread), onWork func(*phpThread), onShutdown func(*phpThread)) { thread.isActive.Store(true) // to avoid race conditions, the thread sets its own hooks on startup @@ -77,7 +77,6 @@ func (thread *phpThread) setHooks(onStartup func(*phpThread), onWork func(*phpTh } // signal to the thread to stop it's current execution and call the onStartup hook - threadsReadyWG.Add(1) close(thread.done) thread.isReady.Store(false) } @@ -110,7 +109,10 @@ func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { if thread.onStartup != nil { thread.onStartup(thread) } - threadsReadyWG.Done() + if threadsAreBooting.Load() { + threadsReadyWG.Done() + threadsReadyWG.Wait() + } } // do the actual work diff --git a/php_threads.go b/php_threads.go index edc2bbfda..c968c20ab 100644 --- a/php_threads.go +++ b/php_threads.go @@ -17,6 +17,7 @@ var ( shutdownWG sync.WaitGroup done chan struct{} threadsAreDone atomic.Bool + threadsAreBooting atomic.Bool ) // reserve a fixed number of PHP threads on the go side @@ -35,6 +36,8 @@ func initPHPThreads(numThreads int) error { // initialize all threads as inactive threadsReadyWG.Add(len(phpThreads)) shutdownWG.Add(len(phpThreads)) + threadsAreBooting.Store(true) + for _, thread := range phpThreads { thread.setInactive() if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { @@ -42,6 +45,8 @@ func initPHPThreads(numThreads int) error { } } threadsReadyWG.Wait() + threadsAreBooting.Store(false) + return nil } diff --git a/php_threads_test.go b/php_threads_test.go index ea31ef287..c8f70b6a7 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -38,7 +38,7 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { for i := 0; i < numThreads; i++ { newThread := getInactivePHPThread() - newThread.setHooks( + newThread.setActive( // onStartup => before the thread is ready func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { @@ -84,7 +84,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { assert.NoError(t, initPHPThreads(numThreads)) for i := 0; i < numThreads; i++ { - getInactivePHPThread().setHooks( + getInactivePHPThread().setActive( // onStartup => fake a request on startup (like a worker would do) func(thread *phpThread) { r, _ := http.NewRequest(http.MethodGet, "sleep.php", nil) @@ -136,7 +136,7 @@ func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { workWG.Add(numThreads) numberOfConversion := i for j := 0; j < numThreads; j++ { - getInactivePHPThread().setHooks( + getInactivePHPThread().setActive( // onStartup => before the thread is ready func(thread *phpThread) { startUpTypes[numberOfConversion].Add(1) diff --git a/worker.go b/worker.go index 31c8d3063..6fd2787eb 100644 --- a/worker.go +++ b/worker.go @@ -79,7 +79,7 @@ func newWorker(o workerOpt) (*worker, error) { } func (worker *worker) startNewThread() { - getInactivePHPThread().setHooks( + getInactivePHPThread().setActive( // onStartup => right before the thread is ready func(thread *phpThread) { thread.worker = worker @@ -185,7 +185,7 @@ func afterWorkerScript(thread *phpThread, exitStatus C.int) { // TODO: make the max restart configurable metrics.StopWorker(thread.worker.fileName, StopReasonRestart) - if c := logger.Check(zapcore.InfoLevel, "restarting"); c != nil { + if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { c.Write(zap.String("worker", thread.worker.fileName)) } return From 3587243f59fe2fbd5fc4df56be80edfec7c606ce Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Thu, 7 Nov 2024 09:25:31 +0100 Subject: [PATCH 23/64] Moves worker request logic to worker.go. --- frankenphp.go | 5 +---- worker.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/frankenphp.go b/frankenphp.go index 15870a9e4..67e3b667c 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -459,10 +459,7 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error // Detect if a worker is available to handle this request if worker, ok := workers[fc.scriptFilename]; ok { - metrics.StartWorkerRequest(fc.scriptFilename) - worker.handleRequest(request) - <-fc.done - metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) + worker.handleRequest(request, fc) return nil } diff --git a/worker.go b/worker.go index 6fd2787eb..37225ddfb 100644 --- a/worker.go +++ b/worker.go @@ -9,6 +9,7 @@ import ( "path/filepath" "sync" "sync/atomic" + "time" "github.com/dunglas/frankenphp/internal/watcher" "go.uber.org/zap" @@ -203,13 +204,17 @@ func afterWorkerScript(thread *phpThread, exitStatus C.int) { }) } -func (worker *worker) handleRequest(r *http.Request) { - worker.threadMutex.RLock() +func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { + metrics.StartWorkerRequest(fc.scriptFilename) + defer metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) + // dispatch requests to all worker threads in order + worker.threadMutex.RLock() for _, thread := range worker.threads { select { case thread.requestChan <- r: worker.threadMutex.RUnlock() + <-fc.done return default: } @@ -218,6 +223,7 @@ func (worker *worker) handleRequest(r *http.Request) { // if no thread was available, fan the request out to all threads // TODO: theoretically there could be autoscaling of threads here worker.requestChan <- r + <-fc.done } //export go_frankenphp_worker_handle_request_start From ec32f0cc55f52dc08c1b61173272d427cf54031e Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Thu, 7 Nov 2024 11:07:41 +0100 Subject: [PATCH 24/64] Removes defer. --- worker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/worker.go b/worker.go index 37225ddfb..1a1563324 100644 --- a/worker.go +++ b/worker.go @@ -206,7 +206,6 @@ func afterWorkerScript(thread *phpThread, exitStatus C.int) { func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { metrics.StartWorkerRequest(fc.scriptFilename) - defer metrics.StopWorkerRequest(fc.scriptFilename, time.Since(fc.startedAt)) // dispatch requests to all worker threads in order worker.threadMutex.RLock() @@ -215,15 +214,18 @@ func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { case thread.requestChan <- r: worker.threadMutex.RUnlock() <-fc.done + metrics.StopWorkerRequest(worker.fileName, time.Since(fc.startedAt)) return default: } } worker.threadMutex.RUnlock() + // if no thread was available, fan the request out to all threads // TODO: theoretically there could be autoscaling of threads here worker.requestChan <- r <-fc.done + metrics.StopWorkerRequest(worker.fileName, time.Since(fc.startedAt)) } //export go_frankenphp_worker_handle_request_start From 4e356989cd2d6597ba2b606403b54c87d4290117 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 11 Nov 2024 19:20:30 +0100 Subject: [PATCH 25/64] Removes call from go to c. --- frankenphp.c | 18 +++++++++++++----- frankenphp.go | 38 ++++++++++++++++---------------------- php_thread.go | 28 ++++++++++++++++++++++++---- php_threads_test.go | 10 +++++++--- worker.go | 7 +++++-- 5 files changed, 65 insertions(+), 36 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index 7a357e093..374607b05 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -823,7 +823,19 @@ static void *php_thread(void *arg) { should_filter_var = default_filter != NULL; // perform work until go signals to stop - while (go_frankenphp_on_thread_work(thread_index)) { + while (true) { + char *scriptName = go_frankenphp_on_thread_work(thread_index); + + // if the script name is NULL, the thread should exit + if (scriptName == NULL) { + break; + } + + // if the script name is not empty, execute the PHP script + if (strlen(scriptName) != 0) { + int exit_status = frankenphp_execute_script(scriptName); + go_frankenphp_after_thread_work(thread_index, exit_status); + } } go_frankenphp_release_known_variable_keys(thread_index); @@ -934,8 +946,6 @@ int frankenphp_request_startup() { int frankenphp_execute_script(char *file_name) { if (frankenphp_request_startup() == FAILURE) { - free(file_name); - file_name = NULL; return FAILURE; } @@ -944,8 +954,6 @@ int frankenphp_execute_script(char *file_name) { zend_file_handle file_handle; zend_stream_init_filename(&file_handle, file_name); - free(file_name); - file_name = NULL; file_handle.primary_script = 1; diff --git a/frankenphp.go b/frankenphp.go index 67e3b667c..d7e4d2992 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -121,7 +121,7 @@ type FrankenPHPContext struct { closed sync.Once responseWriter http.ResponseWriter - exitStatus C.int + exitStatus int done chan interface{} startedAt time.Time @@ -335,7 +335,7 @@ func Init(options ...Option) error { } for i := 0; i < totalThreadCount-workerThreadCount; i++ { - getInactivePHPThread().setActive(nil, handleRequest, nil) + getInactivePHPThread().setActive(nil, handleRequest, afterRequest, nil) } if err := initWorkers(opt.workers); err != nil { @@ -549,30 +549,33 @@ func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string func handleRequest(thread *phpThread) { select { case <-done: + thread.scriptName = "" return case r := <-requestChan: thread.mainRequest = r - - fc, ok := FromContext(r.Context()) - if !ok { - panic(InvalidRequestError) - } - defer func() { - maybeCloseContext(fc) - thread.mainRequest = nil - thread.Unpin() - }() + fc := r.Context().Value(contextKey).(*FrankenPHPContext) if err := updateServerContext(thread, r, true, false); err != nil { rejectRequest(fc.responseWriter, err.Error()) + thread.scriptName = "" + afterRequest(thread, 0) return } - fc.exitStatus = executeScriptClassic(fc.scriptFilename) + // set the scriptName that should be executed + thread.scriptName = fc.scriptFilename } } +func afterRequest(thread *phpThread, exitStatus int) { + fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + maybeCloseContext(fc) + thread.mainRequest = nil + thread.Unpin() +} + func maybeCloseContext(fc *FrankenPHPContext) { fc.closed.Do(func() { close(fc.done) @@ -783,15 +786,6 @@ func go_log(message *C.char, level C.int) { } } -func executeScriptClassic(script string) C.int { - // scriptFilename is freed in frankenphp_execute_script() - exitStatus := C.frankenphp_execute_script(C.CString(script)) - if exitStatus < 0 { - panic(ScriptExecutionError) - } - return exitStatus -} - // ExecuteScriptCLI executes the PHP script passed as parameter. // It returns the exit status code of the script. func ExecuteScriptCLI(script string, args []string) int { diff --git a/php_thread.go b/php_thread.go index 183a31ca6..d19abc31e 100644 --- a/php_thread.go +++ b/php_thread.go @@ -20,6 +20,8 @@ type phpThread struct { requestChan chan *http.Request worker *worker + // the script name for the current request + scriptName string // the index in the phpThreads slice threadIndex int // whether the thread has work assigned to it @@ -30,6 +32,8 @@ type phpThread struct { onStartup func(*phpThread) // the actual work iteration (done in a loop) onWork func(*phpThread) + // after the work iteration is done + onWorkDone func(*phpThread, int) // after the thread is done onShutdown func(*phpThread) // chan to signal the thread to stop the current work iteration @@ -51,6 +55,7 @@ func (thread *phpThread) getActiveRequest() *http.Request { // TODO: Also consider this case: work => inactive => work func (thread *phpThread) setInactive() { thread.isActive.Store(false) + thread.scriptName = "" thread.onWork = func(thread *phpThread) { thread.requestChan = make(chan *http.Request) select { @@ -60,7 +65,7 @@ func (thread *phpThread) setInactive() { } } -func (thread *phpThread) setActive(onStartup func(*phpThread), onWork func(*phpThread), onShutdown func(*phpThread)) { +func (thread *phpThread) setActive(onStartup func(*phpThread), onWork func(*phpThread), onWorkDone func(*phpThread, int), onShutdown func(*phpThread)) { thread.isActive.Store(true) // to avoid race conditions, the thread sets its own hooks on startup @@ -71,6 +76,7 @@ func (thread *phpThread) setActive(onStartup func(*phpThread), onWork func(*phpT thread.onStartup = onStartup thread.onWork = onWork thread.onShutdown = onShutdown + thread.onWorkDone = onWorkDone if thread.onStartup != nil { thread.onStartup(thread) } @@ -95,10 +101,10 @@ func (thread *phpThread) pinCString(s string) *C.char { } //export go_frankenphp_on_thread_work -func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { +func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) *C.char { // first check if FrankPHP is shutting down if threadsAreDone.Load() { - return C.bool(false) + return nil } thread := phpThreads[threadIndex] @@ -117,7 +123,21 @@ func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) C.bool { // do the actual work thread.onWork(thread) - return C.bool(true) + + // return the name of the PHP script that should be executed + return thread.pinCString(thread.scriptName) +} + +//export go_frankenphp_after_thread_work +func go_frankenphp_after_thread_work(threadIndex C.uintptr_t, exitStatus C.int) { + thread := phpThreads[threadIndex] + if exitStatus < 0 { + panic(ScriptExecutionError) + } + if thread.onWorkDone != nil { + thread.onWorkDone(thread, int(exitStatus)) + } + thread.Unpin() } //export go_frankenphp_on_thread_shutdown diff --git a/php_threads_test.go b/php_threads_test.go index c8f70b6a7..2eb251c9a 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -53,6 +53,7 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { workWG.Done() newThread.setInactive() }, + nil, // onShutdown => after the thread is done func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { @@ -91,6 +92,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { r, _ = NewRequestWithContext(r, WithRequestDocumentRoot("/", false)) assert.NoError(t, updateServerContext(thread, r, true, false)) thread.mainRequest = r + thread.scriptName = scriptPath }, // onWork => execute the sleep.php script until we reach maxExecutions func(thread *phpThread) { @@ -103,9 +105,10 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { executionCount++ workWG.Done() executionMutex.Unlock() - - // exit the loop and fail the test if the script fails - if int(executeScriptClassic(scriptPath)) != 0 { + }, + // onWorkDone => check the exit status of the script + func(thread *phpThread, existStatus int) { + if int(existStatus) != 0 { panic("script execution failed: " + scriptPath) } }, @@ -147,6 +150,7 @@ func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { thread.setInactive() workWG.Done() }, + nil, // onShutdown => after the thread is done func(thread *phpThread) { shutdownTypes[numberOfConversion].Add(1) diff --git a/worker.go b/worker.go index 1a1563324..3f50c0113 100644 --- a/worker.go +++ b/worker.go @@ -90,6 +90,7 @@ func (worker *worker) startNewThread() { worker.threadMutex.Lock() worker.threads = append(worker.threads, thread) worker.threadMutex.Unlock() + thread.scriptName = worker.fileName }, // onWork => while the thread is working (in a loop) func(thread *phpThread) { @@ -98,7 +99,9 @@ func (worker *worker) startNewThread() { workerRestartWG.Wait() } beforeWorkerScript(thread) - exitStatus := executeScriptClassic(thread.worker.fileName) + }, + // onWorkDone => after the work iteration is done + func(thread *phpThread, exitStatus int) { afterWorkerScript(thread, exitStatus) }, // onShutdown => after the thread is done @@ -171,7 +174,7 @@ func beforeWorkerScript(thread *phpThread) { } } -func afterWorkerScript(thread *phpThread, exitStatus C.int) { +func afterWorkerScript(thread *phpThread, exitStatus int) { fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.exitStatus = exitStatus From 8a272cba7c382ffb204e75c6f765eba267285208 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 15 Nov 2024 12:58:06 +0100 Subject: [PATCH 26/64] Fixes merge conflict. --- frankenphp.go | 3 +-- php_threads_test.go | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/frankenphp.go b/frankenphp.go index a61b826a7..b5d2bca48 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -558,8 +558,7 @@ func go_sapi_getenv(threadIndex C.uintptr_t, name *C.go_string) *C.char { return phpThreads[threadIndex].pinCString(envValue) } -//export go_handle_request -func go_handle_request(threadIndex C.uintptr_t) bool { +func handleRequest(thread *phpThread) { select { case <-done: thread.scriptName = "" diff --git a/php_threads_test.go b/php_threads_test.go index 2eb251c9a..51228a695 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -12,8 +12,8 @@ import ( ) func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { - logger = zap.NewNop() // the logger needs to not be nil - assert.NoError(t, initPHPThreads(1)) // reserve 1 thread + logger = zap.NewNop() // the logger needs to not be nil + assert.NoError(t, initPHPThreads(1)) // reserve 1 thread assert.Len(t, phpThreads, 1) assert.Equal(t, 0, phpThreads[0].threadIndex) @@ -53,7 +53,7 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { workWG.Done() newThread.setInactive() }, - nil, + nil, // onShutdown => after the thread is done func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { From ecce5d52b45b50994085abd73dfc9d9de56daf56 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 15 Nov 2024 13:00:48 +0100 Subject: [PATCH 27/64] Adds fibers test back in. --- frankenphp_test.go | 17 +++++++++++++++++ testdata/fiber-basic.php | 9 +++++++++ 2 files changed, 26 insertions(+) create mode 100644 testdata/fiber-basic.php diff --git a/frankenphp_test.go b/frankenphp_test.go index 9ca6b1520..436b96b19 100644 --- a/frankenphp_test.go +++ b/frankenphp_test.go @@ -592,6 +592,23 @@ func testFiberNoCgo(t *testing.T, opts *testOptions) { }, opts) } +func TestFiberBasic_module(t *testing.T) { testFiberBasic(t, &testOptions{}) } +func TestFiberBasic_worker(t *testing.T) { + testFiberBasic(t, &testOptions{workerScript: "fiber-basic.php"}) +} +func testFiberBasic(t *testing.T, opts *testOptions) { + runTest(t, func(handler func(http.ResponseWriter, *http.Request), _ *httptest.Server, i int) { + req := httptest.NewRequest("GET", fmt.Sprintf("http://example.com/fiber-basic.php?i=%d", i), nil) + w := httptest.NewRecorder() + handler(w, req) + + resp := w.Result() + body, _ := io.ReadAll(resp.Body) + + assert.Equal(t, string(body), fmt.Sprintf("Fiber %d", i)) + }, opts) +} + func TestRequestHeaders_module(t *testing.T) { testRequestHeaders(t, &testOptions{}) } func TestRequestHeaders_worker(t *testing.T) { testRequestHeaders(t, &testOptions{workerScript: "request-headers.php"}) diff --git a/testdata/fiber-basic.php b/testdata/fiber-basic.php new file mode 100644 index 000000000..bdb52336f --- /dev/null +++ b/testdata/fiber-basic.php @@ -0,0 +1,9 @@ +start(); +}; From 06ebd67cf4b7519db9537775926b9172c608d6ed Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 15 Nov 2024 19:39:30 +0100 Subject: [PATCH 28/64] Refactors new thread loop approach. --- env.go | 84 +++++++++++++++++++++++++++++++++++++++++++ frankenphp.c | 4 +-- frankenphp.go | 88 +++------------------------------------------ php_thread.go | 38 ++++++++++---------- php_threads.go | 1 - php_threads_test.go | 18 ++++++---- worker.go | 87 ++++++++++++++++++++++---------------------- 7 files changed, 163 insertions(+), 157 deletions(-) create mode 100644 env.go diff --git a/env.go b/env.go new file mode 100644 index 000000000..f95c6fd13 --- /dev/null +++ b/env.go @@ -0,0 +1,84 @@ +package frankenphp + +// #include "frankenphp.h" +import "C" +import ( + "os" + "strings" + "unsafe" +) + +//export go_putenv +func go_putenv(str *C.char, length C.int) C.bool { + // Create a byte slice from C string with a specified length + s := C.GoBytes(unsafe.Pointer(str), length) + + // Convert byte slice to string + envString := string(s) + + // Check if '=' is present in the string + if key, val, found := strings.Cut(envString, "="); found { + if os.Setenv(key, val) != nil { + return false // Failure + } + } else { + // No '=', unset the environment variable + if os.Unsetenv(envString) != nil { + return false // Failure + } + } + + return true // Success +} + +//export go_getfullenv +func go_getfullenv(threadIndex C.uintptr_t) (*C.go_string, C.size_t) { + thread := phpThreads[threadIndex] + + env := os.Environ() + goStrings := make([]C.go_string, len(env)*2) + + for i, envVar := range env { + key, val, _ := strings.Cut(envVar, "=") + goStrings[i*2] = C.go_string{C.size_t(len(key)), thread.pinString(key)} + goStrings[i*2+1] = C.go_string{C.size_t(len(val)), thread.pinString(val)} + } + + value := unsafe.SliceData(goStrings) + thread.Pin(value) + + return value, C.size_t(len(env)) +} + +//export go_getenv +func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string) { + thread := phpThreads[threadIndex] + + // Create a byte slice from C string with a specified length + envName := C.GoStringN(name.data, C.int(name.len)) + + // Get the environment variable value + envValue, exists := os.LookupEnv(envName) + if !exists { + // Environment variable does not exist + return false, nil // Return 0 to indicate failure + } + + // Convert Go string to C string + value := &C.go_string{C.size_t(len(envValue)), thread.pinString(envValue)} + thread.Pin(value) + + return true, value // Return 1 to indicate success +} + +//export go_sapi_getenv +func go_sapi_getenv(threadIndex C.uintptr_t, name *C.go_string) *C.char { + envName := C.GoStringN(name.data, C.int(name.len)) + + envValue, exists := os.LookupEnv(envName) + if !exists { + return nil + } + + return phpThreads[threadIndex].pinCString(envValue) +} diff --git a/frankenphp.c b/frankenphp.c index 8492dcda0..0b249e152 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -830,7 +830,7 @@ static void *php_thread(void *arg) { // perform work until go signals to stop while (true) { - char *scriptName = go_frankenphp_on_thread_work(thread_index); + char *scriptName = go_frankenphp_before_script_execution(thread_index); // if the script name is NULL, the thread should exit if (scriptName == NULL) { @@ -840,7 +840,7 @@ static void *php_thread(void *arg) { // if the script name is not empty, execute the PHP script if (strlen(scriptName) != 0) { int exit_status = frankenphp_execute_script(scriptName); - go_frankenphp_after_thread_work(thread_index, exit_status); + go_frankenphp_after_script_execution(thread_index, exit_status); } } diff --git a/frankenphp.go b/frankenphp.go index b5d2bca48..7b7f61b6a 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -476,91 +476,10 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error return nil } -//export go_putenv -func go_putenv(str *C.char, length C.int) C.bool { - // Create a byte slice from C string with a specified length - s := C.GoBytes(unsafe.Pointer(str), length) - - // Convert byte slice to string - envString := string(s) - - // Check if '=' is present in the string - if key, val, found := strings.Cut(envString, "="); found { - if os.Setenv(key, val) != nil { - return false // Failure - } - } else { - // No '=', unset the environment variable - if os.Unsetenv(envString) != nil { - return false // Failure - } - } - - return true // Success -} - -//export go_getfullenv -func go_getfullenv(threadIndex C.uintptr_t) (*C.go_string, C.size_t) { - thread := phpThreads[threadIndex] - - env := os.Environ() - goStrings := make([]C.go_string, len(env)*2) - - for i, envVar := range env { - key, val, _ := strings.Cut(envVar, "=") - k := unsafe.StringData(key) - v := unsafe.StringData(val) - thread.Pin(k) - thread.Pin(v) - - goStrings[i*2] = C.go_string{C.size_t(len(key)), (*C.char)(unsafe.Pointer(k))} - goStrings[i*2+1] = C.go_string{C.size_t(len(val)), (*C.char)(unsafe.Pointer(v))} - } - - value := unsafe.SliceData(goStrings) - thread.Pin(value) - - return value, C.size_t(len(env)) -} - -//export go_getenv -func go_getenv(threadIndex C.uintptr_t, name *C.go_string) (C.bool, *C.go_string) { - thread := phpThreads[threadIndex] - - // Create a byte slice from C string with a specified length - envName := C.GoStringN(name.data, C.int(name.len)) - - // Get the environment variable value - envValue, exists := os.LookupEnv(envName) - if !exists { - // Environment variable does not exist - return false, nil // Return 0 to indicate failure - } - - // Convert Go string to C string - val := unsafe.StringData(envValue) - thread.Pin(val) - value := &C.go_string{C.size_t(len(envValue)), (*C.char)(unsafe.Pointer(val))} - thread.Pin(value) - - return true, value // Return 1 to indicate success -} - -//export go_sapi_getenv -func go_sapi_getenv(threadIndex C.uintptr_t, name *C.go_string) *C.char { - envName := C.GoStringN(name.data, C.int(name.len)) - - envValue, exists := os.LookupEnv(envName) - if !exists { - return nil - } - - return phpThreads[threadIndex].pinCString(envValue) -} - func handleRequest(thread *phpThread) { select { case <-done: + // no script should be executed if the server is shutting down thread.scriptName = "" return @@ -570,8 +489,10 @@ func handleRequest(thread *phpThread) { if err := updateServerContext(thread, r, true, false); err != nil { rejectRequest(fc.responseWriter, err.Error()) - thread.scriptName = "" afterRequest(thread, 0) + thread.Unpin() + // no script should be executed if the request was rejected + thread.scriptName = "" return } @@ -585,7 +506,6 @@ func afterRequest(thread *phpThread, exitStatus int) { fc.exitStatus = exitStatus maybeCloseContext(fc) thread.mainRequest = nil - thread.Unpin() } func maybeCloseContext(fc *FrankenPHPContext) { diff --git a/php_thread.go b/php_thread.go index d19abc31e..5c00959b0 100644 --- a/php_thread.go +++ b/php_thread.go @@ -1,8 +1,5 @@ package frankenphp -// #include -// #include -// #include // #include "frankenphp.h" import "C" import ( @@ -31,9 +28,9 @@ type phpThread struct { // right before the first work iteration onStartup func(*phpThread) // the actual work iteration (done in a loop) - onWork func(*phpThread) + beforeScriptExecution func(*phpThread) // after the work iteration is done - onWorkDone func(*phpThread, int) + afterScriptExecution func(*phpThread, int) // after the thread is done onShutdown func(*phpThread) // chan to signal the thread to stop the current work iteration @@ -56,7 +53,7 @@ func (thread *phpThread) getActiveRequest() *http.Request { func (thread *phpThread) setInactive() { thread.isActive.Store(false) thread.scriptName = "" - thread.onWork = func(thread *phpThread) { + thread.beforeScriptExecution = func(thread *phpThread) { thread.requestChan = make(chan *http.Request) select { case <-done: @@ -65,7 +62,12 @@ func (thread *phpThread) setInactive() { } } -func (thread *phpThread) setActive(onStartup func(*phpThread), onWork func(*phpThread), onWorkDone func(*phpThread, int), onShutdown func(*phpThread)) { +func (thread *phpThread) setActive( + onStartup func(*phpThread), + beforeScriptExecution func(*phpThread), + afterScriptExecution func(*phpThread, int), + onShutdown func(*phpThread), +) { thread.isActive.Store(true) // to avoid race conditions, the thread sets its own hooks on startup @@ -74,9 +76,9 @@ func (thread *phpThread) setActive(onStartup func(*phpThread), onWork func(*phpT thread.onShutdown(thread) } thread.onStartup = onStartup - thread.onWork = onWork + thread.beforeScriptExecution = beforeScriptExecution thread.onShutdown = onShutdown - thread.onWorkDone = onWorkDone + thread.afterScriptExecution = afterScriptExecution if thread.onStartup != nil { thread.onStartup(thread) } @@ -100,9 +102,9 @@ func (thread *phpThread) pinCString(s string) *C.char { return thread.pinString(s + "\x00") } -//export go_frankenphp_on_thread_work -func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) *C.char { - // first check if FrankPHP is shutting down +//export go_frankenphp_before_script_execution +func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { + // returning nil signals the thread to stop if threadsAreDone.Load() { return nil } @@ -121,21 +123,21 @@ func go_frankenphp_on_thread_work(threadIndex C.uintptr_t) *C.char { } } - // do the actual work - thread.onWork(thread) + // execute a hook before the script is executed + thread.beforeScriptExecution(thread) // return the name of the PHP script that should be executed return thread.pinCString(thread.scriptName) } -//export go_frankenphp_after_thread_work -func go_frankenphp_after_thread_work(threadIndex C.uintptr_t, exitStatus C.int) { +//export go_frankenphp_after_script_execution +func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C.int) { thread := phpThreads[threadIndex] if exitStatus < 0 { panic(ScriptExecutionError) } - if thread.onWorkDone != nil { - thread.onWorkDone(thread, int(exitStatus)) + if thread.afterScriptExecution != nil { + thread.afterScriptExecution(thread, int(exitStatus)) } thread.Unpin() } diff --git a/php_threads.go b/php_threads.go index c968c20ab..11826ba5a 100644 --- a/php_threads.go +++ b/php_threads.go @@ -1,6 +1,5 @@ package frankenphp -// #include // #include "frankenphp.h" import "C" import ( diff --git a/php_threads_test.go b/php_threads_test.go index 51228a695..b290e0c77 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -45,7 +45,7 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { readyThreads.Add(1) } }, - // onWork => while the thread is running (we stop here immediately) + // beforeScriptExecution => we stop here immediately func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { workingThreads.Add(1) @@ -53,7 +53,10 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { workWG.Done() newThread.setInactive() }, - nil, + // afterScriptExecution => no script is executed, we shouldn't reach here + func(thread *phpThread, exitStatus int) { + panic("hook afterScriptExecution should not be called here") + }, // onShutdown => after the thread is done func(thread *phpThread) { if thread.threadIndex == newThread.threadIndex { @@ -94,7 +97,7 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { thread.mainRequest = r thread.scriptName = scriptPath }, - // onWork => execute the sleep.php script until we reach maxExecutions + // beforeScriptExecution => execute the sleep.php script until we reach maxExecutions func(thread *phpThread) { executionMutex.Lock() if executionCount >= maxExecutions { @@ -106,9 +109,9 @@ func TestSleep10000TimesIn100Threads(t *testing.T) { workWG.Done() executionMutex.Unlock() }, - // onWorkDone => check the exit status of the script - func(thread *phpThread, existStatus int) { - if int(existStatus) != 0 { + // afterScriptExecution => check the exit status of the script + func(thread *phpThread, exitStatus int) { + if int(exitStatus) != 0 { panic("script execution failed: " + scriptPath) } }, @@ -144,12 +147,13 @@ func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { func(thread *phpThread) { startUpTypes[numberOfConversion].Add(1) }, - // onWork => while the thread is running + // beforeScriptExecution => while the thread is running func(thread *phpThread) { workTypes[numberOfConversion].Add(1) thread.setInactive() workWG.Done() }, + // afterScriptExecution => we don't execute a script nil, // onShutdown => after the thread is done func(thread *phpThread) { diff --git a/worker.go b/worker.go index 53e03c85d..01ef153aa 100644 --- a/worker.go +++ b/worker.go @@ -1,6 +1,5 @@ package frankenphp -// #include // #include "frankenphp.h" import "C" import ( @@ -80,39 +79,6 @@ func newWorker(o workerOpt) (*worker, error) { return w, nil } -func (worker *worker) startNewThread() { - getInactivePHPThread().setActive( - // onStartup => right before the thread is ready - func(thread *phpThread) { - thread.worker = worker - thread.requestChan = make(chan *http.Request) - metrics.ReadyWorker(worker.fileName) - thread.backoff = newExponentialBackoff() - worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() - thread.scriptName = worker.fileName - }, - // onWork => while the thread is working (in a loop) - func(thread *phpThread) { - if watcherIsEnabled && workersAreRestarting.Load() { - workerShutdownWG.Done() - workerRestartWG.Wait() - } - beforeWorkerScript(thread) - }, - // onWorkDone => after the work iteration is done - func(thread *phpThread, exitStatus int) { - afterWorkerScript(thread, exitStatus) - }, - // onShutdown => after the thread is done - func(thread *phpThread) { - thread.worker = nil - thread.backoff = nil - }, - ) -} - func stopWorkers() { close(workersDone) } @@ -129,7 +95,7 @@ func restartWorkers() { workerShutdownWG.Add(worker.num) } workersAreRestarting.Store(true) - close(workersDone) + stopWorkers() workerShutdownWG.Wait() workersDone = make(chan interface{}) workersAreRestarting.Store(false) @@ -143,10 +109,42 @@ func getDirectoriesToWatch(workerOpts []workerOpt) []string { return directoriesToWatch } -func beforeWorkerScript(thread *phpThread) { - worker := thread.worker +func (worker *worker) startNewThread() { + getInactivePHPThread().setActive( + // onStartup => right before the thread is ready + func(thread *phpThread) { + thread.worker = worker + thread.scriptName = worker.fileName + thread.requestChan = make(chan *http.Request) + thread.backoff = newExponentialBackoff() + worker.threadMutex.Lock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() + metrics.ReadyWorker(worker.fileName) + }, + // beforeScriptExecution => set up the worker with a fake request + func(thread *phpThread) { + worker.beforeScript(thread) + }, + // afterScriptExecution => tear down the worker + func(thread *phpThread, exitStatus int) { + worker.afterScript(thread, exitStatus) + }, + // onShutdown => after the thread is done + func(thread *phpThread) { + thread.worker = nil + thread.backoff = nil + }, + ) +} + +func (worker *worker) beforeScript(thread *phpThread) { + // if we are restarting due to file watching, wait for all workers to finish first + if watcherIsEnabled && workersAreRestarting.Load() { + workerShutdownWG.Done() + workerRestartWG.Wait() + } - // if we are restarting the worker, reset the exponential failure backoff thread.backoff.reset() metrics.StartWorker(worker.fileName) @@ -175,36 +173,35 @@ func beforeWorkerScript(thread *phpThread) { } } -func afterWorkerScript(thread *phpThread, exitStatus int) { +func (worker *worker) afterScript(thread *phpThread, exitStatus int) { fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.exitStatus = exitStatus defer func() { maybeCloseContext(fc) thread.mainRequest = nil - thread.Unpin() }() // on exit status 0 we just run the worker script again if fc.exitStatus == 0 { // TODO: make the max restart configurable - metrics.StopWorker(thread.worker.fileName, StopReasonRestart) + metrics.StopWorker(worker.fileName, StopReasonRestart) if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName)) + c.Write(zap.String("worker", worker.fileName)) } return } // on exit status 1 we apply an exponential backoff when restarting - metrics.StopWorker(thread.worker.fileName, StopReasonCrash) + metrics.StopWorker(worker.fileName, StopReasonCrash) thread.backoff.trigger(func(failureCount int) { // if we end up here, the worker has not been up for backoff*2 // this is probably due to a syntax error or another fatal error if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", thread.worker.fileName)) + panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) } - logger.Warn("many consecutive worker failures", zap.String("worker", thread.worker.fileName), zap.Int("failures", failureCount)) + logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) }) } From c811f4a167cde72eed5651cfc7ed3cfea859b262 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 16 Nov 2024 16:57:45 +0100 Subject: [PATCH 29/64] Removes redundant check. --- worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker.go b/worker.go index 01ef153aa..1b245b2da 100644 --- a/worker.go +++ b/worker.go @@ -140,7 +140,7 @@ func (worker *worker) startNewThread() { func (worker *worker) beforeScript(thread *phpThread) { // if we are restarting due to file watching, wait for all workers to finish first - if watcherIsEnabled && workersAreRestarting.Load() { + if workersAreRestarting.Load() { workerShutdownWG.Done() workerRestartWG.Wait() } From 6bd047a4cc6b69c1acfa45b819786e0706259d96 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 16 Nov 2024 16:58:00 +0100 Subject: [PATCH 30/64] Adds compareAndSwap. --- php_thread.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/php_thread.go b/php_thread.go index 5c00959b0..a8d64ce41 100644 --- a/php_thread.go +++ b/php_thread.go @@ -111,8 +111,7 @@ func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] // if the thread is not ready, set it up - if !thread.isReady.Load() { - thread.isReady.Store(true) + if thread.isReady.CompareAndSwap(false, true) { thread.done = make(chan struct{}) if thread.onStartup != nil { thread.onStartup(thread) From 55ad8ba8bcde8937374ee849302a291fcda21220 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sun, 17 Nov 2024 22:39:57 +0100 Subject: [PATCH 31/64] Refactor: removes global waitgroups and uses a 'thread state' abstraction instead. --- frankenphp.c | 2 + php_thread.go | 47 ++++++++++---------- php_threads.go | 71 +++++++++++++++-------------- php_threads_test.go | 4 +- thread_state.go | 103 +++++++++++++++++++++++++++++++++++++++++++ thread_state_test.go | 43 ++++++++++++++++++ worker.go | 40 +++++++++-------- 7 files changed, 233 insertions(+), 77 deletions(-) create mode 100644 thread_state.go create mode 100644 thread_state_test.go diff --git a/frankenphp.c b/frankenphp.c index 0b249e152..73a0dc0be 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -828,6 +828,8 @@ static void *php_thread(void *arg) { cfg_get_string("filter.default", &default_filter); should_filter_var = default_filter != NULL; + go_frankenphp_on_thread_startup(thread_index); + // perform work until go signals to stop while (true) { char *scriptName = go_frankenphp_before_script_execution(thread_index); diff --git a/php_thread.go b/php_thread.go index a8d64ce41..bd260f6c7 100644 --- a/php_thread.go +++ b/php_thread.go @@ -39,6 +39,8 @@ type phpThread struct { backoff *exponentialBackoff // known $_SERVER key names knownVariableKeys map[string]*C.zend_string + // the state handler + state *threadStateHandler } func (thread *phpThread) getActiveRequest() *http.Request { @@ -49,16 +51,11 @@ func (thread *phpThread) getActiveRequest() *http.Request { return thread.mainRequest } -// TODO: Also consider this case: work => inactive => work func (thread *phpThread) setInactive() { - thread.isActive.Store(false) thread.scriptName = "" - thread.beforeScriptExecution = func(thread *phpThread) { - thread.requestChan = make(chan *http.Request) - select { - case <-done: - case <-thread.done: - } + // TODO: handle this in a state machine + if !thread.state.is(stateShuttingDown) { + thread.state.set(stateInactive) } } @@ -68,8 +65,6 @@ func (thread *phpThread) setActive( afterScriptExecution func(*phpThread, int), onShutdown func(*phpThread), ) { - thread.isActive.Store(true) - // to avoid race conditions, the thread sets its own hooks on startup thread.onStartup = func(thread *phpThread) { if thread.onShutdown != nil { @@ -83,10 +78,7 @@ func (thread *phpThread) setActive( thread.onStartup(thread) } } - - // signal to the thread to stop it's current execution and call the onStartup hook - close(thread.done) - thread.isReady.Store(false) + thread.state.set(stateActive) } // Pin a string that is not null-terminated @@ -102,24 +94,31 @@ func (thread *phpThread) pinCString(s string) *C.char { return thread.pinString(s + "\x00") } +//export go_frankenphp_on_thread_startup +func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { + phpThreads[threadIndex].setInactive() +} + //export go_frankenphp_before_script_execution func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { + thread := phpThreads[threadIndex] + + // if the state is inactive, wait for it to be active + if thread.state.is(stateInactive) { + thread.state.waitFor(stateActive, stateShuttingDown) + } + // returning nil signals the thread to stop - if threadsAreDone.Load() { + if thread.state.is(stateShuttingDown) { return nil } - thread := phpThreads[threadIndex] - // if the thread is not ready, set it up - if thread.isReady.CompareAndSwap(false, true) { - thread.done = make(chan struct{}) + // if the thread is not ready yet, set it up + if !thread.state.is(stateReady) { + thread.state.set(stateReady) if thread.onStartup != nil { thread.onStartup(thread) } - if threadsAreBooting.Load() { - threadsReadyWG.Done() - threadsReadyWG.Wait() - } } // execute a hook before the script is executed @@ -148,5 +147,5 @@ func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { if thread.onShutdown != nil { thread.onShutdown(thread) } - shutdownWG.Done() + thread.state.set(stateDone) } diff --git a/php_threads.go b/php_threads.go index 11826ba5a..9ef71fde8 100644 --- a/php_threads.go +++ b/php_threads.go @@ -5,73 +5,78 @@ import "C" import ( "fmt" "sync" - "sync/atomic" ) var ( - phpThreads []*phpThread - terminationWG sync.WaitGroup - mainThreadShutdownWG sync.WaitGroup - threadsReadyWG sync.WaitGroup - shutdownWG sync.WaitGroup - done chan struct{} - threadsAreDone atomic.Bool - threadsAreBooting atomic.Bool + phpThreads []*phpThread + done chan struct{} + mainThreadState *threadStateHandler ) // reserve a fixed number of PHP threads on the go side func initPHPThreads(numThreads int) error { - threadsReadyWG = sync.WaitGroup{} - threadsAreDone.Store(false) done = make(chan struct{}) phpThreads = make([]*phpThread, numThreads) for i := 0; i < numThreads; i++ { - phpThreads[i] = &phpThread{threadIndex: i} + phpThreads[i] = &phpThread{ + threadIndex: i, + state: &threadStateHandler{currentState: stateBooting}, + } } if err := startMainThread(numThreads); err != nil { return err } // initialize all threads as inactive - threadsReadyWG.Add(len(phpThreads)) - shutdownWG.Add(len(phpThreads)) - threadsAreBooting.Store(true) + ready := sync.WaitGroup{} + ready.Add(len(phpThreads)) for _, thread := range phpThreads { - thread.setInactive() - if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { - panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) - } + go func() { + if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { + panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) + } + thread.state.waitFor(stateInactive) + ready.Done() + }() } - threadsReadyWG.Wait() - threadsAreBooting.Store(false) + + ready.Wait() return nil } func drainPHPThreads() { - threadsAreDone.Store(true) + doneWG := sync.WaitGroup{} + doneWG.Add(len(phpThreads)) + for _, thread := range phpThreads { + thread.state.set(stateShuttingDown) + } close(done) - shutdownWG.Wait() - mainThreadShutdownWG.Done() - terminationWG.Wait() + for _, thread := range phpThreads { + go func(thread *phpThread) { + thread.state.waitFor(stateDone) + doneWG.Done() + }(thread) + } + doneWG.Wait() + mainThreadState.set(stateShuttingDown) + mainThreadState.waitFor(stateDone) phpThreads = nil } func startMainThread(numThreads int) error { - threadsReadyWG.Add(1) - mainThreadShutdownWG.Add(1) - terminationWG.Add(1) + mainThreadState = &threadStateHandler{currentState: stateBooting} if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { return MainThreadCreationError } - threadsReadyWG.Wait() + mainThreadState.waitFor(stateActive) return nil } func getInactivePHPThread() *phpThread { for _, thread := range phpThreads { - if !thread.isActive.Load() { + if thread.state.is(stateInactive) { return thread } } @@ -80,11 +85,11 @@ func getInactivePHPThread() *phpThread { //export go_frankenphp_main_thread_is_ready func go_frankenphp_main_thread_is_ready() { - threadsReadyWG.Done() - mainThreadShutdownWG.Wait() + mainThreadState.set(stateActive) + mainThreadState.waitFor(stateShuttingDown) } //export go_frankenphp_shutdown_main_thread func go_frankenphp_shutdown_main_thread() { - terminationWG.Done() + mainThreadState.set(stateDone) } diff --git a/php_threads_test.go b/php_threads_test.go index b290e0c77..ab85c783f 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -17,7 +17,7 @@ func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { assert.Len(t, phpThreads, 1) assert.Equal(t, 0, phpThreads[0].threadIndex) - assert.False(t, phpThreads[0].isActive.Load()) + assert.True(t, phpThreads[0].state.is(stateInactive)) assert.Nil(t, phpThreads[0].worker) drainPHPThreads() @@ -76,7 +76,7 @@ func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { // This test calls sleep() 10.000 times for 1ms in 100 PHP threads. func TestSleep10000TimesIn100Threads(t *testing.T) { - logger = zap.NewNop() // the logger needs to not be nil + logger, _ = zap.NewDevelopment() // the logger needs to not be nil numThreads := 100 maxExecutions := 10000 executionMutex := sync.Mutex{} diff --git a/thread_state.go b/thread_state.go new file mode 100644 index 000000000..00540610b --- /dev/null +++ b/thread_state.go @@ -0,0 +1,103 @@ +package frankenphp + +import ( + "slices" + "sync" +) + +type threadState int + +const ( + stateBooting threadState = iota + stateInactive + stateActive + stateReady + stateWorking + stateShuttingDown + stateDone + stateRestarting +) + +type threadStateHandler struct { + currentState threadState + mu sync.RWMutex + subscribers []stateSubscriber +} + +type stateSubscriber struct { + states []threadState + ch chan struct{} + yieldFor *sync.WaitGroup +} + +func (h *threadStateHandler) is(state threadState) bool { + h.mu.RLock() + defer h.mu.RUnlock() + return h.currentState == state +} + +func (h *threadStateHandler) get() threadState { + h.mu.RLock() + defer h.mu.RUnlock() + return h.currentState +} + +func (h *threadStateHandler) set(nextState threadState) { + h.mu.Lock() + defer h.mu.Unlock() + if h.currentState == nextState { + // TODO: do we return here or inform subscribers? + // TODO: should we ever reach here? + return + } + + h.currentState = nextState + + if len(h.subscribers) == 0 { + return + } + + newSubscribers := []stateSubscriber{} + // TODO: do we even need multiple subscribers? + // notify subscribers to the state change + for _, sub := range h.subscribers { + if !slices.Contains(sub.states, nextState) { + newSubscribers = append(newSubscribers, sub) + continue + } + close(sub.ch) + // yield for the subscriber + if sub.yieldFor != nil { + defer sub.yieldFor.Wait() + } + } + h.subscribers = newSubscribers +} + +// wait for the thread to reach a certain state +func (h *threadStateHandler) waitFor(states ...threadState) { + h.waitForStates(states, nil) +} + +// make the thread yield to a WaitGroup once it reaches the state +// this makes sure all threads are in sync both ways +func (h *threadStateHandler) waitForAndYield(yieldFor *sync.WaitGroup, states ...threadState) { + h.waitForStates(states, yieldFor) +} + +// subscribe to a state and wait until the thread reaches it +func (h *threadStateHandler) waitForStates(states []threadState, yieldFor *sync.WaitGroup) { + h.mu.Lock() + if slices.Contains(states, h.currentState) { + h.mu.Unlock() + return + } + sub := stateSubscriber{ + states: states, + ch: make(chan struct{}), + yieldFor: yieldFor, + } + h.subscribers = append(h.subscribers, sub) + h.mu.Unlock() + <-sub.ch +} diff --git a/thread_state_test.go b/thread_state_test.go new file mode 100644 index 000000000..10d42635a --- /dev/null +++ b/thread_state_test.go @@ -0,0 +1,43 @@ +package frankenphp + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func TestYieldToEachOtherViaThreadStates(t *testing.T) { + threadState := &threadStateHandler{currentState: stateBooting} + + go func() { + threadState.waitFor(stateInactive) + assert.True(t, threadState.is(stateInactive)) + threadState.set(stateActive) + }() + + threadState.set(stateInactive) + threadState.waitFor(stateActive) + assert.True(t, threadState.is(stateActive)) +} + +func TestYieldToAWaitGroupPassedByThreadState(t *testing.T) { + logger, _ = zap.NewDevelopment() + threadState := &threadStateHandler{currentState: stateBooting} + hasYielded := false + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + threadState.set(stateInactive) + threadState.waitForAndYield(&wg, stateActive) + hasYielded = true + wg.Done() + }() + + threadState.waitFor(stateInactive) + threadState.set(stateActive) + // the state should be 'ready' since we are also yielding to the WaitGroup + assert.True(t, hasYielded) +} diff --git a/worker.go b/worker.go index 1b245b2da..60722867c 100644 --- a/worker.go +++ b/worker.go @@ -8,7 +8,6 @@ import ( "net/http" "path/filepath" "sync" - "sync/atomic" "time" "github.com/dunglas/frankenphp/internal/watcher" @@ -26,12 +25,9 @@ type worker struct { } var ( - workers map[string]*worker - workersDone chan interface{} - watcherIsEnabled bool - workersAreRestarting atomic.Bool - workerRestartWG sync.WaitGroup - workerShutdownWG sync.WaitGroup + workers map[string]*worker + workersDone chan interface{} + watcherIsEnabled bool ) func initWorkers(opt []workerOpt) error { @@ -89,16 +85,25 @@ func drainWorkers() { } func restartWorkers() { - workerRestartWG.Add(1) - defer workerRestartWG.Done() + restart := sync.WaitGroup{} + restart.Add(1) + ready := sync.WaitGroup{} for _, worker := range workers { - workerShutdownWG.Add(worker.num) + worker.threadMutex.RLock() + ready.Add(len(worker.threads)) + for _, thread := range worker.threads { + thread.state.set(stateRestarting) + go func(thread *phpThread) { + thread.state.waitForAndYield(&restart, stateReady) + ready.Done() + }(thread) + } + worker.threadMutex.RUnlock() } - workersAreRestarting.Store(true) stopWorkers() - workerShutdownWG.Wait() + ready.Wait() workersDone = make(chan interface{}) - workersAreRestarting.Store(false) + restart.Done() } func getDirectoriesToWatch(workerOpts []workerOpt) []string { @@ -139,10 +144,9 @@ func (worker *worker) startNewThread() { } func (worker *worker) beforeScript(thread *phpThread) { - // if we are restarting due to file watching, wait for all workers to finish first - if workersAreRestarting.Load() { - workerShutdownWG.Done() - workerRestartWG.Wait() + // if we are restarting due to file watching, set the state back to ready + if thread.state.is(stateRestarting) { + thread.state.set(stateReady) } thread.backoff.reset() @@ -245,7 +249,7 @@ func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { } // execute opcache_reset if the restart was triggered by the watcher - if watcherIsEnabled && workersAreRestarting.Load() && !executePHPFunction("opcache_reset") { + if watcherIsEnabled && thread.state.is(stateRestarting) && !executePHPFunction("opcache_reset") { logger.Error("failed to call opcache_reset") } From 01ed92bc3becc127639827e5de702fc5051263db Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sun, 17 Nov 2024 22:58:31 +0100 Subject: [PATCH 32/64] Removes unnecessary method. --- thread_state.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/thread_state.go b/thread_state.go index 00540610b..a66947254 100644 --- a/thread_state.go +++ b/thread_state.go @@ -36,12 +36,6 @@ func (h *threadStateHandler) is(state threadState) bool { return h.currentState == state } -func (h *threadStateHandler) get() threadState { - h.mu.RLock() - defer h.mu.RUnlock() - return h.currentState -} - func (h *threadStateHandler) set(nextState threadState) { h.mu.Lock() defer h.mu.Unlock() From 790cccc1641e555ca4a97d787243cb1a6ab1d8fe Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sun, 17 Nov 2024 23:15:31 +0100 Subject: [PATCH 33/64] Updates comment. --- thread_state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thread_state_test.go b/thread_state_test.go index 10d42635a..d9ff5fcdb 100644 --- a/thread_state_test.go +++ b/thread_state_test.go @@ -38,6 +38,6 @@ func TestYieldToAWaitGroupPassedByThreadState(t *testing.T) { threadState.waitFor(stateInactive) threadState.set(stateActive) - // the state should be 'ready' since we are also yielding to the WaitGroup + // 'set' should have yielded to the wait group assert.True(t, hasYielded) } From 0dd26051493dffbdddd6455f5c21f7109f5a12b1 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 18 Nov 2024 09:29:17 +0100 Subject: [PATCH 34/64] Removes unnecessary booleans. --- php_thread.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/php_thread.go b/php_thread.go index bd260f6c7..1692c6d3c 100644 --- a/php_thread.go +++ b/php_thread.go @@ -5,7 +5,6 @@ import "C" import ( "net/http" "runtime" - "sync/atomic" "unsafe" ) @@ -21,10 +20,6 @@ type phpThread struct { scriptName string // the index in the phpThreads slice threadIndex int - // whether the thread has work assigned to it - isActive atomic.Bool - // whether the thread is ready for work - isReady atomic.Bool // right before the first work iteration onStartup func(*phpThread) // the actual work iteration (done in a loop) @@ -33,8 +28,6 @@ type phpThread struct { afterScriptExecution func(*phpThread, int) // after the thread is done onShutdown func(*phpThread) - // chan to signal the thread to stop the current work iteration - done chan struct{} // exponential backoff for worker failures backoff *exponentialBackoff // known $_SERVER key names From 60a66b4128820f99f2e967e2189cc539ca6476d1 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sun, 24 Nov 2024 15:27:37 +0100 Subject: [PATCH 35/64] test --- thread_state.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/thread_state.go b/thread_state.go index a66947254..5a79cd23f 100644 --- a/thread_state.go +++ b/thread_state.go @@ -24,6 +24,13 @@ type threadStateHandler struct { subscribers []stateSubscriber } +type threadStateMachine interface { + onStartup(*phpThread) + beforeScriptExecution(*phpThread) + afterScriptExecution(*phpThread, int) + onShutdown(*phpThread) +} + type stateSubscriber struct { states []threadState ch chan struct{} From 4719fa8ea15a41b461e7fde99a13870c8d56c14d Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 25 Nov 2024 22:09:35 +0100 Subject: [PATCH 36/64] First state machine steps. --- php_thread.go | 41 +++++----- thread_state.go | 14 ++-- worker.go | 67 +--------------- worker_state_machine.go | 166 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 93 deletions(-) create mode 100644 worker_state_machine.go diff --git a/php_thread.go b/php_thread.go index 1692c6d3c..f0e25eb33 100644 --- a/php_thread.go +++ b/php_thread.go @@ -34,6 +34,7 @@ type phpThread struct { knownVariableKeys map[string]*C.zend_string // the state handler state *threadStateHandler + stateMachine *workerStateMachine } func (thread *phpThread) getActiveRequest() *http.Request { @@ -89,34 +90,38 @@ func (thread *phpThread) pinCString(s string) *C.char { //export go_frankenphp_on_thread_startup func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { - phpThreads[threadIndex].setInactive() + phpThreads[threadIndex].state.set(stateInactive) } //export go_frankenphp_before_script_execution func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] + thread.state.set(stateReady) // if the state is inactive, wait for it to be active - if thread.state.is(stateInactive) { - thread.state.waitFor(stateActive, stateShuttingDown) - } + //if thread.state.is(stateInactive) { + // thread.state.waitFor(stateActive, stateShuttingDown) + //} // returning nil signals the thread to stop - if thread.state.is(stateShuttingDown) { - return nil - } + //if thread.state.is(stateShuttingDown) { + // return nil + //} // if the thread is not ready yet, set it up - if !thread.state.is(stateReady) { - thread.state.set(stateReady) - if thread.onStartup != nil { - thread.onStartup(thread) - } - } + //if !thread.state.is(stateReady) { + // thread.state.set(stateReady) + // if thread.onStartup != nil { + // thread.onStartup(thread) + // } + //} // execute a hook before the script is executed - thread.beforeScriptExecution(thread) + //thread.beforeScriptExecution(thread) + if thread.stateMachine.done { + return nil + } // return the name of the PHP script that should be executed return thread.pinCString(thread.scriptName) } @@ -127,18 +132,10 @@ func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C. if exitStatus < 0 { panic(ScriptExecutionError) } - if thread.afterScriptExecution != nil { - thread.afterScriptExecution(thread, int(exitStatus)) - } thread.Unpin() } //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { - thread := phpThreads[threadIndex] - thread.Unpin() - if thread.onShutdown != nil { - thread.onShutdown(thread) - } thread.state.set(stateDone) } diff --git a/thread_state.go b/thread_state.go index 5a79cd23f..ea1ba833b 100644 --- a/thread_state.go +++ b/thread_state.go @@ -12,7 +12,7 @@ const ( stateInactive stateActive stateReady - stateWorking + stateBusy stateShuttingDown stateDone stateRestarting @@ -25,10 +25,8 @@ type threadStateHandler struct { } type threadStateMachine interface { - onStartup(*phpThread) - beforeScriptExecution(*phpThread) - afterScriptExecution(*phpThread, int) - onShutdown(*phpThread) + handleState(state threadState) + isDone() bool } type stateSubscriber struct { @@ -43,6 +41,12 @@ func (h *threadStateHandler) is(state threadState) bool { return h.currentState == state } +func (h *threadStateHandler) get(state threadState) threadState { + h.mu.RLock() + defer h.mu.RUnlock() + return h.currentState +} + func (h *threadStateHandler) set(nextState threadState) { h.mu.Lock() defer h.mu.Unlock() diff --git a/worker.go b/worker.go index b5b7195ca..fe6d0031f 100644 --- a/worker.go +++ b/worker.go @@ -24,6 +24,7 @@ type worker struct { threadMutex sync.RWMutex } + var ( workers map[string]*worker workersDone chan interface{} @@ -143,72 +144,6 @@ func (worker *worker) startNewThread() { ) } -func (worker *worker) beforeScript(thread *phpThread) { - // if we are restarting due to file watching, set the state back to ready - if thread.state.is(stateRestarting) { - thread.state.set(stateReady) - } - - thread.backoff.reset() - metrics.StartWorker(worker.fileName) - - // Create a dummy request to set up the worker - r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) - if err != nil { - panic(err) - } - - r, err = NewRequestWithContext( - r, - WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), - WithRequestPreparedEnv(worker.env), - ) - if err != nil { - panic(err) - } - - if err := updateServerContext(thread, r, true, false); err != nil { - panic(err) - } - - thread.mainRequest = r - if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { - c.Write(zap.String("worker", worker.fileName), zap.Int("thread", thread.threadIndex)) - } -} - -func (worker *worker) afterScript(thread *phpThread, exitStatus int) { - fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) - fc.exitStatus = exitStatus - - defer func() { - maybeCloseContext(fc) - thread.mainRequest = nil - }() - - // on exit status 0 we just run the worker script again - if fc.exitStatus == 0 { - // TODO: make the max restart configurable - metrics.StopWorker(worker.fileName, StopReasonRestart) - - if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { - c.Write(zap.String("worker", worker.fileName)) - } - return - } - - // on exit status 1 we apply an exponential backoff when restarting - metrics.StopWorker(worker.fileName, StopReasonCrash) - thread.backoff.trigger(func(failureCount int) { - // if we end up here, the worker has not been up for backoff*2 - // this is probably due to a syntax error or another fatal error - if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) - } - logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) - }) -} - func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { metrics.StartWorkerRequest(fc.scriptFilename) diff --git a/worker_state_machine.go b/worker_state_machine.go new file mode 100644 index 000000000..33a2fcf52 --- /dev/null +++ b/worker_state_machine.go @@ -0,0 +1,166 @@ +package frankenphp + +import ( + "net/http" + "sync" + "strconv" + + "golang.uber.org/zap" +) + +type workerStateMachine struct { + state *threadStateHandler + thread *phpThread + worker *worker + isDone bool +} + + +func (w *workerStateMachine) isDone() threadState { + return w.isDone +} + +func (w *workerStateMachine) handleState(nextState threadState) { + previousState := w.state.get() + + switch previousState { + case stateBooting: + switch nextState { + case stateInactive: + w.state.set(stateInactive) + // waiting for external signal to start + w.state.waitFor(stateReady, stateShuttingDown) + return + } + case stateInactive: + switch nextState { + case stateReady: + w.state.set(stateReady) + beforeScript(w.thread) + return + case stateShuttingDown: + w.shutdown() + return + } + case stateReady: + switch nextState { + case stateBusy: + w.state.set(stateBusy) + return + case stateShuttingDown: + w.shutdown() + return + } + case stateBusy: + afterScript(w.thread, w.worker) + switch nextState { + case stateReady: + w.state.set(stateReady) + beforeScript(w.thread, w.worker) + return + case stateShuttingDown: + w.shutdown() + return + case stateRestarting: + w.state.set(stateRestarting) + return + } + case stateShuttingDown: + switch nextState { + case stateDone: + w.thread.Unpin() + w.state.set(stateDone) + return + case stateRestarting: + w.state.set(stateRestarting) + return + } + case stateDone: + panic("Worker is done") + case stateRestarting: + switch nextState { + case stateReady: + // wait for external ready signal + w.state.waitFor(stateReady) + return + case stateShuttingDown: + w.shutdown() + return + } + } + + panic("Invalid state transition from", zap.Int("from", int(previousState)), zap.Int("to", int(nextState))) +} + +func (w *workerStateMachine) shutdown() { + w.thread.scriptName = "" + workerStateMachine.done = true + w.thread.state.set(stateShuttingDown) +} + +func beforeScript(thread *phpThread, worker *worker) { + thread.worker = worker + // if we are restarting due to file watching, set the state back to ready + if thread.state.is(stateRestarting) { + thread.state.set(stateReady) + } + + thread.backoff.reset() + metrics.StartWorker(worker.fileName) + + // Create a dummy request to set up the worker + r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) + if err != nil { + panic(err) + } + + r, err = NewRequestWithContext( + r, + WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), + WithRequestPreparedEnv(worker.env), + ) + if err != nil { + panic(err) + } + + if err := updateServerContext(thread, r, true, false); err != nil { + panic(err) + } + + thread.mainRequest = r + if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { + c.Write(zap.String("worker", worker.fileName), zap.Int("thread", thread.threadIndex)) + } +} + +func (worker *worker) afterScript(thread *phpThread, exitStatus int) { + fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + + defer func() { + maybeCloseContext(fc) + thread.mainRequest = nil + }() + + // on exit status 0 we just run the worker script again + if fc.exitStatus == 0 { + // TODO: make the max restart configurable + metrics.StopWorker(worker.fileName, StopReasonRestart) + + if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { + c.Write(zap.String("worker", worker.fileName)) + } + return + } + + // on exit status 1 we apply an exponential backoff when restarting + metrics.StopWorker(worker.fileName, StopReasonCrash) + thread.backoff.trigger(func(failureCount int) { + // if we end up here, the worker has not been up for backoff*2 + // this is probably due to a syntax error or another fatal error + if !watcherIsEnabled { + panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) + } + logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) + }) +} From f72e8cbb8ac7320b489da5196818f73c1f6abb89 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 6 Dec 2024 19:46:54 +0100 Subject: [PATCH 37/64] Splits threads. --- frankenphp.go | 21 ----- php_regular_thread.go | 128 +++++++++++++++++++++++++ php_thread.go | 53 ++++------- php_worker_thread.go | 204 ++++++++++++++++++++++++++++++++++++++++ thread_state.go | 4 - worker.go | 45 +-------- worker_state_machine.go | 166 -------------------------------- 7 files changed, 350 insertions(+), 271 deletions(-) create mode 100644 php_regular_thread.go create mode 100644 php_worker_thread.go delete mode 100644 worker_state_machine.go diff --git a/frankenphp.go b/frankenphp.go index 7b7f61b6a..ca3e79874 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -477,28 +477,7 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error } func handleRequest(thread *phpThread) { - select { - case <-done: - // no script should be executed if the server is shutting down - thread.scriptName = "" - return - case r := <-requestChan: - thread.mainRequest = r - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - - if err := updateServerContext(thread, r, true, false); err != nil { - rejectRequest(fc.responseWriter, err.Error()) - afterRequest(thread, 0) - thread.Unpin() - // no script should be executed if the request was rejected - thread.scriptName = "" - return - } - - // set the scriptName that should be executed - thread.scriptName = fc.scriptFilename - } } func afterRequest(thread *phpThread, exitStatus int) { diff --git a/php_regular_thread.go b/php_regular_thread.go new file mode 100644 index 000000000..bc11447a1 --- /dev/null +++ b/php_regular_thread.go @@ -0,0 +1,128 @@ +package frankenphp + +import ( + "net/http" + "sync" + "strconv" + + "go.uber.org/zap" +) + +type phpRegularThread struct { + state *threadStateHandler + thread *phpThread + worker *worker + isDone bool + pinner *runtime.Pinner + getActiveRequest *http.Request + knownVariableKeys map[string]*C.zend_string +} + +// this is done once +func (thread *phpWorkerThread) onStartup(){ + // do nothing +} + +func (thread *phpWorkerThread) pinner() *runtime.Pinner { + return thread.pinner +} + +func (thread *phpWorkerThread) getActiveRequest() *http.Request { + return thread.activeRequest +} + +// return the name of the script or an empty string if no script should be executed +func (thread *phpWorkerThread) beforeScriptExecution() string { + currentState := w.state.get() + switch currentState { + case stateInactive: + thread.state.waitFor(stateActive, stateShuttingDown) + return thread.beforeScriptExecution() + case stateShuttingDown: + return "" + case stateReady, stateActive: + return waitForScriptExecution(thread) + } +} + +// return true if the worker should continue to run +func (thread *phpWorkerThread) afterScriptExecution() bool { + tearDownWorkerScript(thread, thread.worker) + currentState := w.state.get() + switch currentState { + case stateDrain: + thread.requestChan = make(chan *http.Request) + return true + case stateShuttingDown: + return false + } + return true +} + +func (thread *phpWorkerThread) onShutdown(){ + state.set(stateDone) +} + +func waitForScriptExecution(thread *phpThread) string { + select { + case <-done: + // no script should be executed if the server is shutting down + thread.scriptName = "" + return + + case r := <-requestChan: + thread.mainRequest = r + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + + if err := updateServerContext(thread, r, true, false); err != nil { + rejectRequest(fc.responseWriter, err.Error()) + afterRequest(thread, 0) + thread.Unpin() + // no script should be executed if the request was rejected + return "" + } + + // set the scriptName that should be executed + return fc.scriptFilename + } +} + +func tearDownWorkerScript(thread *phpThread, exitStatus int) { + fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + + defer func() { + maybeCloseContext(fc) + thread.mainRequest = nil + }() + + // on exit status 0 we just run the worker script again + worker := thread.worker + if fc.exitStatus == 0 { + // TODO: make the max restart configurable + metrics.StopWorker(worker.fileName, StopReasonRestart) + + if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { + c.Write(zap.String("worker", worker.fileName)) + } + return + } + + // on exit status 1 we apply an exponential backoff when restarting + metrics.StopWorker(worker.fileName, StopReasonCrash) + thread.backoff.trigger(func(failureCount int) { + // if we end up here, the worker has not been up for backoff*2 + // this is probably due to a syntax error or another fatal error + if !watcherIsEnabled { + panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) + } + logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) + }) +} + +func (thread *phpWorkerThread) getKnownVariableKeys() map[string]*C.zend_string{ + return thread.knownVariableKeys +} +func (thread *phpWorkerThread) setKnownVariableKeys(map[string]*C.zend_string){ + thread.knownVariableKeys = knownVariableKeys +} \ No newline at end of file diff --git a/php_thread.go b/php_thread.go index f0e25eb33..bac7707c7 100644 --- a/php_thread.go +++ b/php_thread.go @@ -8,6 +8,17 @@ import ( "unsafe" ) +type phpThread interface { + onStartup() + beforeScriptExecution() string + afterScriptExecution(exitStatus int) bool + onShutdown() + pinner() *runtime.Pinner + getActiveRequest() *http.Request + getKnownVariableKeys() map[string]*C.zend_string + setKnownVariableKeys(map[string]*C.zend_string) +} + type phpThread struct { runtime.Pinner @@ -45,14 +56,6 @@ func (thread *phpThread) getActiveRequest() *http.Request { return thread.mainRequest } -func (thread *phpThread) setInactive() { - thread.scriptName = "" - // TODO: handle this in a state machine - if !thread.state.is(stateShuttingDown) { - thread.state.set(stateInactive) - } -} - func (thread *phpThread) setActive( onStartup func(*phpThread), beforeScriptExecution func(*phpThread), @@ -90,40 +93,15 @@ func (thread *phpThread) pinCString(s string) *C.char { //export go_frankenphp_on_thread_startup func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { - phpThreads[threadIndex].state.set(stateInactive) + phpThreads[threadIndex].stateMachine.onStartup() } //export go_frankenphp_before_script_execution func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] - thread.state.set(stateReady) - - // if the state is inactive, wait for it to be active - //if thread.state.is(stateInactive) { - // thread.state.waitFor(stateActive, stateShuttingDown) - //} - - // returning nil signals the thread to stop - //if thread.state.is(stateShuttingDown) { - // return nil - //} - - // if the thread is not ready yet, set it up - //if !thread.state.is(stateReady) { - // thread.state.set(stateReady) - // if thread.onStartup != nil { - // thread.onStartup(thread) - // } - //} - - // execute a hook before the script is executed - //thread.beforeScriptExecution(thread) - - if thread.stateMachine.done { - return nil - } + scriptName := thread.stateMachine.beforeScriptExecution() // return the name of the PHP script that should be executed - return thread.pinCString(thread.scriptName) + return thread.pinCString(scriptName) } //export go_frankenphp_after_script_execution @@ -132,10 +110,11 @@ func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C. if exitStatus < 0 { panic(ScriptExecutionError) } + thread.stateMachine.afterScriptExecution(int(exitStatus)) thread.Unpin() } //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { - thread.state.set(stateDone) + thread.stateMachine.onShutdown() } diff --git a/php_worker_thread.go b/php_worker_thread.go new file mode 100644 index 000000000..c9421151f --- /dev/null +++ b/php_worker_thread.go @@ -0,0 +1,204 @@ +package frankenphp + +import ( + "net/http" + "sync" + "strconv" + + "go.uber.org/zap" +) + +type phpWorkerThread struct { + state *threadStateHandler + thread *phpThread + worker *worker + isDone bool + pinner *runtime.Pinner + mainRequest *http.Request + workerRequest *http.Request + knownVariableKeys map[string]*C.zend_string +} + +// this is done once +func (thread *phpWorkerThread) onStartup(){ + thread.requestChan = make(chan *http.Request) + thread.backoff = newExponentialBackoff() + thread.worker.threadMutex.Lock() + thread.worker.threads = append(worker.threads, thread) + thread.worker.threadMutex.Unlock() +} + +func (thread *phpWorkerThread) pinner() *runtime.Pinner { + return thread.pinner +} + +func (thread *phpWorkerThread) getActiveRequest() *http.Request { + if thread.workerRequest != nil { + return thread.workerRequest + } + + return thread.mainRequest +} + +// return the name of the script or an empty string if no script should be executed +func (thread *phpWorkerThread) beforeScriptExecution() string { + currentState := w.state.get() + switch currentState { + case stateInactive: + thread.state.waitFor(stateActive, stateShuttingDown) + return thread.beforeScriptExecution() + case stateShuttingDown: + return "" + case stateRestarting: + thread.state.waitFor(stateReady, stateShuttingDown) + setUpWorkerScript(thread, thread.worker) + return thread.worker.fileName + case stateReady, stateActive: + setUpWorkerScript(w.thread, w.worker) + return thread.worker.fileName + } +} + +func (thread *phpWorkerThread) waitForWorkerRequest() bool { + + if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { + c.Write(zap.String("worker", thread.worker.fileName)) + } + + if !thread.state.is(stateReady) { + metrics.ReadyWorker(w.worker.fileName) + thread.state.set(stateReady) + } + + var r *http.Request + select { + case <-workersDone: + if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { + c.Write(zap.String("worker", thread.worker.fileName)) + } + + // execute opcache_reset if the restart was triggered by the watcher + if watcherIsEnabled && thread.state.is(stateRestarting) { + C.frankenphp_reset_opcache() + } + + return false + case r = <-thread.requestChan: + case r = <-thread.worker.requestChan: + } + + thread.workerRequest = r + + if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { + c.Write(zap.String("worker", thread.worker.fileName), zap.String("url", r.RequestURI)) + } + + if err := updateServerContext(thread, r, false, true); err != nil { + // Unexpected error + if c := logger.Check(zapcore.DebugLevel, "unexpected error"); c != nil { + c.Write(zap.String("worker", thread.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) + } + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + rejectRequest(fc.responseWriter, err.Error()) + maybeCloseContext(fc) + thread.workerRequest = nil + thread.Unpin() + + return go_frankenphp_worker_handle_request_start(threadIndex) + } + return true +} + +// return true if the worker should continue to run +func (thread *phpWorkerThread) afterScriptExecution() bool { + tearDownWorkerScript(thread, thread.worker) + currentState := w.state.get() + switch currentState { + case stateDrain: + thread.requestChan = make(chan *http.Request) + return true + } + case stateShuttingDown: + return false + } + return true +} + +func (thread *phpWorkerThread) onShutdown(){ + state.set(stateDone) +} + +func setUpWorkerScript(thread *phpThread, worker *worker) { + thread.worker = worker + // if we are restarting due to file watching, set the state back to ready + if thread.state.is(stateRestarting) { + thread.state.set(stateReady) + } + + thread.backoff.reset() + metrics.StartWorker(worker.fileName) + + // Create a dummy request to set up the worker + r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) + if err != nil { + panic(err) + } + + r, err = NewRequestWithContext( + r, + WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), + WithRequestPreparedEnv(worker.env), + ) + if err != nil { + panic(err) + } + + if err := updateServerContext(thread, r, true, false); err != nil { + panic(err) + } + + thread.mainRequest = r + if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { + c.Write(zap.String("worker", worker.fileName), zap.Int("thread", thread.threadIndex)) + } +} + +func tearDownWorkerScript(thread *phpThread, exitStatus int) { + fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + + defer func() { + maybeCloseContext(fc) + thread.mainRequest = nil + }() + + // on exit status 0 we just run the worker script again + worker := thread.worker + if fc.exitStatus == 0 { + // TODO: make the max restart configurable + metrics.StopWorker(worker.fileName, StopReasonRestart) + + if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { + c.Write(zap.String("worker", worker.fileName)) + } + return + } + + // on exit status 1 we apply an exponential backoff when restarting + metrics.StopWorker(worker.fileName, StopReasonCrash) + thread.backoff.trigger(func(failureCount int) { + // if we end up here, the worker has not been up for backoff*2 + // this is probably due to a syntax error or another fatal error + if !watcherIsEnabled { + panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) + } + logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) + }) +} + +func (thread *phpWorkerThread) getKnownVariableKeys() map[string]*C.zend_string{ + return thread.knownVariableKeys +} +func (thread *phpWorkerThread) setKnownVariableKeys(map[string]*C.zend_string){ + thread.knownVariableKeys = knownVariableKeys +} \ No newline at end of file diff --git a/thread_state.go b/thread_state.go index ea1ba833b..f26f7d653 100644 --- a/thread_state.go +++ b/thread_state.go @@ -24,10 +24,6 @@ type threadStateHandler struct { subscribers []stateSubscriber } -type threadStateMachine interface { - handleState(state threadState) - isDone() bool -} type stateSubscriber struct { states []threadState diff --git a/worker.go b/worker.go index fe6d0031f..35c862298 100644 --- a/worker.go +++ b/worker.go @@ -170,49 +170,8 @@ func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { //export go_frankenphp_worker_handle_request_start func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { - thread := phpThreads[threadIndex] - - if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName)) - } - - var r *http.Request - select { - case <-workersDone: - if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName)) - } - - // execute opcache_reset if the restart was triggered by the watcher - if watcherIsEnabled && thread.state.is(stateRestarting) { - C.frankenphp_reset_opcache() - } - - return C.bool(false) - case r = <-thread.requestChan: - case r = <-thread.worker.requestChan: - } - - thread.workerRequest = r - - if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName), zap.String("url", r.RequestURI)) - } - - if err := updateServerContext(thread, r, false, true); err != nil { - // Unexpected error - if c := logger.Check(zapcore.DebugLevel, "unexpected error"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) - } - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - rejectRequest(fc.responseWriter, err.Error()) - maybeCloseContext(fc) - thread.workerRequest = nil - thread.Unpin() - - return go_frankenphp_worker_handle_request_start(threadIndex) - } - return C.bool(true) + thread := phpWorkerThread(phpThreads[threadIndex]) + return C.bool(thread.stateMachine.waitForWorkerRequest(stateReady)) } //export go_frankenphp_finish_worker_request diff --git a/worker_state_machine.go b/worker_state_machine.go deleted file mode 100644 index 33a2fcf52..000000000 --- a/worker_state_machine.go +++ /dev/null @@ -1,166 +0,0 @@ -package frankenphp - -import ( - "net/http" - "sync" - "strconv" - - "golang.uber.org/zap" -) - -type workerStateMachine struct { - state *threadStateHandler - thread *phpThread - worker *worker - isDone bool -} - - -func (w *workerStateMachine) isDone() threadState { - return w.isDone -} - -func (w *workerStateMachine) handleState(nextState threadState) { - previousState := w.state.get() - - switch previousState { - case stateBooting: - switch nextState { - case stateInactive: - w.state.set(stateInactive) - // waiting for external signal to start - w.state.waitFor(stateReady, stateShuttingDown) - return - } - case stateInactive: - switch nextState { - case stateReady: - w.state.set(stateReady) - beforeScript(w.thread) - return - case stateShuttingDown: - w.shutdown() - return - } - case stateReady: - switch nextState { - case stateBusy: - w.state.set(stateBusy) - return - case stateShuttingDown: - w.shutdown() - return - } - case stateBusy: - afterScript(w.thread, w.worker) - switch nextState { - case stateReady: - w.state.set(stateReady) - beforeScript(w.thread, w.worker) - return - case stateShuttingDown: - w.shutdown() - return - case stateRestarting: - w.state.set(stateRestarting) - return - } - case stateShuttingDown: - switch nextState { - case stateDone: - w.thread.Unpin() - w.state.set(stateDone) - return - case stateRestarting: - w.state.set(stateRestarting) - return - } - case stateDone: - panic("Worker is done") - case stateRestarting: - switch nextState { - case stateReady: - // wait for external ready signal - w.state.waitFor(stateReady) - return - case stateShuttingDown: - w.shutdown() - return - } - } - - panic("Invalid state transition from", zap.Int("from", int(previousState)), zap.Int("to", int(nextState))) -} - -func (w *workerStateMachine) shutdown() { - w.thread.scriptName = "" - workerStateMachine.done = true - w.thread.state.set(stateShuttingDown) -} - -func beforeScript(thread *phpThread, worker *worker) { - thread.worker = worker - // if we are restarting due to file watching, set the state back to ready - if thread.state.is(stateRestarting) { - thread.state.set(stateReady) - } - - thread.backoff.reset() - metrics.StartWorker(worker.fileName) - - // Create a dummy request to set up the worker - r, err := http.NewRequest(http.MethodGet, filepath.Base(worker.fileName), nil) - if err != nil { - panic(err) - } - - r, err = NewRequestWithContext( - r, - WithRequestDocumentRoot(filepath.Dir(worker.fileName), false), - WithRequestPreparedEnv(worker.env), - ) - if err != nil { - panic(err) - } - - if err := updateServerContext(thread, r, true, false); err != nil { - panic(err) - } - - thread.mainRequest = r - if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { - c.Write(zap.String("worker", worker.fileName), zap.Int("thread", thread.threadIndex)) - } -} - -func (worker *worker) afterScript(thread *phpThread, exitStatus int) { - fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) - fc.exitStatus = exitStatus - - defer func() { - maybeCloseContext(fc) - thread.mainRequest = nil - }() - - // on exit status 0 we just run the worker script again - if fc.exitStatus == 0 { - // TODO: make the max restart configurable - metrics.StopWorker(worker.fileName, StopReasonRestart) - - if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { - c.Write(zap.String("worker", worker.fileName)) - } - return - } - - // on exit status 1 we apply an exponential backoff when restarting - metrics.StopWorker(worker.fileName, StopReasonCrash) - thread.backoff.trigger(func(failureCount int) { - // if we end up here, the worker has not been up for backoff*2 - // this is probably due to a syntax error or another fatal error - if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) - } - logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) - }) -} From d20e70677bc27021974f16b07235d5da0f0a0ee3 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 6 Dec 2024 23:07:19 +0100 Subject: [PATCH 38/64] Minimal working implementation with broken tests. --- frankenphp.go | 12 +-- php_inactive_thread.go | 49 ++++++++++++ php_regular_thread.go | 106 +++++++++--------------- php_thread.go | 83 +++++-------------- php_threads.go | 9 ++- php_worker_thread.go | 178 +++++++++++++++++++++++++---------------- thread_state.go | 24 ++++-- thread_state_test.go | 4 +- worker.go | 74 ++--------------- 9 files changed, 251 insertions(+), 288 deletions(-) create mode 100644 php_inactive_thread.go diff --git a/frankenphp.go b/frankenphp.go index ca3e79874..10210b2f9 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -335,7 +335,8 @@ func Init(options ...Option) error { } for i := 0; i < totalThreadCount-workerThreadCount; i++ { - getInactivePHPThread().setActive(nil, handleRequest, afterRequest, nil) + thread := getInactivePHPThread() + convertToRegularThread(thread) } if err := initWorkers(opt.workers); err != nil { @@ -480,13 +481,6 @@ func handleRequest(thread *phpThread) { } -func afterRequest(thread *phpThread, exitStatus int) { - fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) - fc.exitStatus = exitStatus - maybeCloseContext(fc) - thread.mainRequest = nil -} - func maybeCloseContext(fc *FrankenPHPContext) { fc.closed.Do(func() { close(fc.done) @@ -538,7 +532,7 @@ func go_apache_request_headers(threadIndex C.uintptr_t, hasActiveRequest bool) ( if !hasActiveRequest { // worker mode, not handling a request - mfc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) + mfc := thread.getActiveRequest().Context().Value(contextKey).(*FrankenPHPContext) if c := mfc.logger.Check(zapcore.DebugLevel, "apache_request_headers() called in non-HTTP context"); c != nil { c.Write(zap.String("worker", mfc.scriptFilename)) diff --git a/php_inactive_thread.go b/php_inactive_thread.go new file mode 100644 index 000000000..ff816f7b9 --- /dev/null +++ b/php_inactive_thread.go @@ -0,0 +1,49 @@ +package frankenphp + +import ( + "net/http" + "strconv" +) + +type phpInactiveThread struct { + state *stateHandler +} + +func convertToInactiveThread(thread *phpThread) { + thread.handler = &phpInactiveThread{state: thread.state} +} + +func (t *phpInactiveThread) isReadyToTransition() bool { + return true +} + +// this is done once +func (thread *phpInactiveThread) onStartup(){ + // do nothing +} + +func (thread *phpInactiveThread) getActiveRequest() *http.Request { + panic("idle threads have no requests") +} + +func (thread *phpInactiveThread) beforeScriptExecution() string { + thread.state.set(stateInactive) + return "" +} + +func (thread *phpInactiveThread) afterScriptExecution(exitStatus int) bool { + // wait for external signal to start or shut down + thread.state.waitFor(stateActive, stateShuttingDown) + switch thread.state.get() { + case stateActive: + return true + case stateShuttingDown: + return false + } + panic("unexpected state: "+strconv.Itoa(int(thread.state.get()))) +} + +func (thread *phpInactiveThread) onShutdown(){ + thread.state.set(stateDone) +} + diff --git a/php_regular_thread.go b/php_regular_thread.go index bc11447a1..001f5304f 100644 --- a/php_regular_thread.go +++ b/php_regular_thread.go @@ -1,39 +1,42 @@ package frankenphp +// #include "frankenphp.h" +import "C" import ( "net/http" - "sync" - "strconv" - - "go.uber.org/zap" ) type phpRegularThread struct { - state *threadStateHandler + state *stateHandler thread *phpThread - worker *worker - isDone bool - pinner *runtime.Pinner - getActiveRequest *http.Request - knownVariableKeys map[string]*C.zend_string + activeRequest *http.Request } -// this is done once -func (thread *phpWorkerThread) onStartup(){ - // do nothing +func convertToRegularThread(thread *phpThread) { + thread.handler = &phpRegularThread{ + thread: thread, + state: thread.state, + } + thread.handler.onStartup() + thread.state.set(stateActive) } -func (thread *phpWorkerThread) pinner() *runtime.Pinner { - return thread.pinner +func (t *phpRegularThread) isReadyToTransition() bool { + return false +} + +// this is done once +func (thread *phpRegularThread) onStartup(){ + // do nothing } -func (thread *phpWorkerThread) getActiveRequest() *http.Request { +func (thread *phpRegularThread) getActiveRequest() *http.Request { return thread.activeRequest } // return the name of the script or an empty string if no script should be executed -func (thread *phpWorkerThread) beforeScriptExecution() string { - currentState := w.state.get() +func (thread *phpRegularThread) beforeScriptExecution() string { + currentState := thread.state.get() switch currentState { case stateInactive: thread.state.waitFor(stateActive, stateShuttingDown) @@ -43,15 +46,16 @@ func (thread *phpWorkerThread) beforeScriptExecution() string { case stateReady, stateActive: return waitForScriptExecution(thread) } + return "" } // return true if the worker should continue to run -func (thread *phpWorkerThread) afterScriptExecution() bool { - tearDownWorkerScript(thread, thread.worker) - currentState := w.state.get() +func (thread *phpRegularThread) afterScriptExecution(exitStatus int) bool { + thread.afterRequest(exitStatus) + + currentState := thread.state.get() switch currentState { case stateDrain: - thread.requestChan = make(chan *http.Request) return true case stateShuttingDown: return false @@ -59,25 +63,24 @@ func (thread *phpWorkerThread) afterScriptExecution() bool { return true } -func (thread *phpWorkerThread) onShutdown(){ - state.set(stateDone) +func (thread *phpRegularThread) onShutdown(){ + thread.state.set(stateDone) } -func waitForScriptExecution(thread *phpThread) string { +func waitForScriptExecution(thread *phpRegularThread) string { select { case <-done: // no script should be executed if the server is shutting down - thread.scriptName = "" - return + return "" case r := <-requestChan: - thread.mainRequest = r + thread.activeRequest = r fc := r.Context().Value(contextKey).(*FrankenPHPContext) - if err := updateServerContext(thread, r, true, false); err != nil { + if err := updateServerContext(thread.thread, r, true, false); err != nil { rejectRequest(fc.responseWriter, err.Error()) - afterRequest(thread, 0) - thread.Unpin() + thread.afterRequest(0) + thread.thread.Unpin() // no script should be executed if the request was rejected return "" } @@ -87,42 +90,9 @@ func waitForScriptExecution(thread *phpThread) string { } } -func tearDownWorkerScript(thread *phpThread, exitStatus int) { - fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) +func (thread *phpRegularThread) afterRequest(exitStatus int) { + fc := thread.activeRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.exitStatus = exitStatus - - defer func() { - maybeCloseContext(fc) - thread.mainRequest = nil - }() - - // on exit status 0 we just run the worker script again - worker := thread.worker - if fc.exitStatus == 0 { - // TODO: make the max restart configurable - metrics.StopWorker(worker.fileName, StopReasonRestart) - - if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { - c.Write(zap.String("worker", worker.fileName)) - } - return - } - - // on exit status 1 we apply an exponential backoff when restarting - metrics.StopWorker(worker.fileName, StopReasonCrash) - thread.backoff.trigger(func(failureCount int) { - // if we end up here, the worker has not been up for backoff*2 - // this is probably due to a syntax error or another fatal error - if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) - } - logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) - }) -} - -func (thread *phpWorkerThread) getKnownVariableKeys() map[string]*C.zend_string{ - return thread.knownVariableKeys + maybeCloseContext(fc) + thread.activeRequest = nil } -func (thread *phpWorkerThread) setKnownVariableKeys(map[string]*C.zend_string){ - thread.knownVariableKeys = knownVariableKeys -} \ No newline at end of file diff --git a/php_thread.go b/php_thread.go index bac7707c7..4d1fd5b3a 100644 --- a/php_thread.go +++ b/php_thread.go @@ -8,73 +8,34 @@ import ( "unsafe" ) -type phpThread interface { - onStartup() - beforeScriptExecution() string - afterScriptExecution(exitStatus int) bool - onShutdown() - pinner() *runtime.Pinner - getActiveRequest() *http.Request - getKnownVariableKeys() map[string]*C.zend_string - setKnownVariableKeys(map[string]*C.zend_string) -} - +// representation of the actual underlying PHP thread +// identified by the index in the phpThreads slice type phpThread struct { runtime.Pinner - mainRequest *http.Request - workerRequest *http.Request - requestChan chan *http.Request - worker *worker - - // the script name for the current request - scriptName string - // the index in the phpThreads slice threadIndex int - // right before the first work iteration - onStartup func(*phpThread) - // the actual work iteration (done in a loop) - beforeScriptExecution func(*phpThread) - // after the work iteration is done - afterScriptExecution func(*phpThread, int) - // after the thread is done - onShutdown func(*phpThread) - // exponential backoff for worker failures - backoff *exponentialBackoff - // known $_SERVER key names knownVariableKeys map[string]*C.zend_string - // the state handler - state *threadStateHandler - stateMachine *workerStateMachine + requestChan chan *http.Request + handler threadHandler + state *stateHandler } -func (thread *phpThread) getActiveRequest() *http.Request { - if thread.workerRequest != nil { - return thread.workerRequest - } +// interface that defines how the callbacks from the C thread should be handled +type threadHandler interface { + onStartup() + beforeScriptExecution() string + afterScriptExecution(exitStatus int) bool + onShutdown() + getActiveRequest() *http.Request + isReadyToTransition() bool +} - return thread.mainRequest +func (thread *phpThread) getActiveRequest() *http.Request { + return thread.handler.getActiveRequest() } -func (thread *phpThread) setActive( - onStartup func(*phpThread), - beforeScriptExecution func(*phpThread), - afterScriptExecution func(*phpThread, int), - onShutdown func(*phpThread), -) { - // to avoid race conditions, the thread sets its own hooks on startup - thread.onStartup = func(thread *phpThread) { - if thread.onShutdown != nil { - thread.onShutdown(thread) - } - thread.onStartup = onStartup - thread.beforeScriptExecution = beforeScriptExecution - thread.onShutdown = onShutdown - thread.afterScriptExecution = afterScriptExecution - if thread.onStartup != nil { - thread.onStartup(thread) - } - } +func (thread *phpThread) setHandler(handler threadHandler) { + thread.handler = handler thread.state.set(stateActive) } @@ -93,13 +54,13 @@ func (thread *phpThread) pinCString(s string) *C.char { //export go_frankenphp_on_thread_startup func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { - phpThreads[threadIndex].stateMachine.onStartup() + phpThreads[threadIndex].handler.onStartup() } //export go_frankenphp_before_script_execution func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] - scriptName := thread.stateMachine.beforeScriptExecution() + scriptName := thread.handler.beforeScriptExecution() // return the name of the PHP script that should be executed return thread.pinCString(scriptName) } @@ -110,11 +71,11 @@ func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C. if exitStatus < 0 { panic(ScriptExecutionError) } - thread.stateMachine.afterScriptExecution(int(exitStatus)) + thread.handler.afterScriptExecution(int(exitStatus)) thread.Unpin() } //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { - thread.stateMachine.onShutdown() + phpThreads[threadIndex].handler.onShutdown() } diff --git a/php_threads.go b/php_threads.go index 9ef71fde8..67486a259 100644 --- a/php_threads.go +++ b/php_threads.go @@ -10,7 +10,7 @@ import ( var ( phpThreads []*phpThread done chan struct{} - mainThreadState *threadStateHandler + mainThreadState *stateHandler ) // reserve a fixed number of PHP threads on the go side @@ -20,8 +20,9 @@ func initPHPThreads(numThreads int) error { for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{ threadIndex: i, - state: &threadStateHandler{currentState: stateBooting}, + state: newStateHandler(), } + convertToInactiveThread(phpThreads[i]) } if err := startMainThread(numThreads); err != nil { return err @@ -66,7 +67,7 @@ func drainPHPThreads() { } func startMainThread(numThreads int) error { - mainThreadState = &threadStateHandler{currentState: stateBooting} + mainThreadState = newStateHandler() if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { return MainThreadCreationError } @@ -76,7 +77,7 @@ func startMainThread(numThreads int) error { func getInactivePHPThread() *phpThread { for _, thread := range phpThreads { - if thread.state.is(stateInactive) { + if thread.handler.isReadyToTransition() { return thread } } diff --git a/php_worker_thread.go b/php_worker_thread.go index c9421151f..0a87f3de0 100644 --- a/php_worker_thread.go +++ b/php_worker_thread.go @@ -1,141 +1,147 @@ package frankenphp +// #include "frankenphp.h" +import "C" import ( "net/http" - "sync" - "strconv" + "path/filepath" + "fmt" "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) type phpWorkerThread struct { - state *threadStateHandler + state *stateHandler thread *phpThread worker *worker - isDone bool - pinner *runtime.Pinner mainRequest *http.Request workerRequest *http.Request - knownVariableKeys map[string]*C.zend_string + backoff *exponentialBackoff } -// this is done once -func (thread *phpWorkerThread) onStartup(){ - thread.requestChan = make(chan *http.Request) - thread.backoff = newExponentialBackoff() - thread.worker.threadMutex.Lock() - thread.worker.threads = append(worker.threads, thread) - thread.worker.threadMutex.Unlock() +func convertToWorkerThread(thread *phpThread, worker *worker) { + thread.handler = &phpWorkerThread{ + state: thread.state, + thread: thread, + worker: worker, + } + thread.handler.onStartup() + thread.state.set(stateActive) } -func (thread *phpWorkerThread) pinner() *runtime.Pinner { - return thread.pinner +// this is done once +func (handler *phpWorkerThread) onStartup(){ + handler.thread.requestChan = make(chan *http.Request) + handler.backoff = newExponentialBackoff() + handler.worker.threadMutex.Lock() + handler.worker.threads = append(handler.worker.threads, handler.thread) + handler.worker.threadMutex.Unlock() } -func (thread *phpWorkerThread) getActiveRequest() *http.Request { - if thread.workerRequest != nil { - return thread.workerRequest +func (handler *phpWorkerThread) getActiveRequest() *http.Request { + if handler.workerRequest != nil { + return handler.workerRequest } - return thread.mainRequest + return handler.mainRequest +} + +func (t *phpWorkerThread) isReadyToTransition() bool { + return false } // return the name of the script or an empty string if no script should be executed -func (thread *phpWorkerThread) beforeScriptExecution() string { - currentState := w.state.get() +func (handler *phpWorkerThread) beforeScriptExecution() string { + currentState := handler.state.get() switch currentState { case stateInactive: - thread.state.waitFor(stateActive, stateShuttingDown) - return thread.beforeScriptExecution() + handler.state.waitFor(stateActive, stateShuttingDown) + return handler.beforeScriptExecution() case stateShuttingDown: return "" case stateRestarting: - thread.state.waitFor(stateReady, stateShuttingDown) - setUpWorkerScript(thread, thread.worker) - return thread.worker.fileName + handler.state.set(stateYielding) + handler.state.waitFor(stateReady, stateShuttingDown) + return handler.beforeScriptExecution() case stateReady, stateActive: - setUpWorkerScript(w.thread, w.worker) - return thread.worker.fileName + setUpWorkerScript(handler, handler.worker) + return handler.worker.fileName } + // TODO: panic? + return "" } -func (thread *phpWorkerThread) waitForWorkerRequest() bool { +func (handler *phpWorkerThread) waitForWorkerRequest() bool { if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName)) + c.Write(zap.String("worker", handler.worker.fileName)) } - if !thread.state.is(stateReady) { - metrics.ReadyWorker(w.worker.fileName) - thread.state.set(stateReady) + if !handler.state.is(stateReady) { + metrics.ReadyWorker(handler.worker.fileName) + handler.state.set(stateReady) } var r *http.Request select { case <-workersDone: if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName)) + c.Write(zap.String("worker", handler.worker.fileName)) } // execute opcache_reset if the restart was triggered by the watcher - if watcherIsEnabled && thread.state.is(stateRestarting) { + if watcherIsEnabled && handler.state.is(stateRestarting) { C.frankenphp_reset_opcache() } return false - case r = <-thread.requestChan: - case r = <-thread.worker.requestChan: + case r = <-handler.thread.requestChan: + case r = <-handler.worker.requestChan: } - thread.workerRequest = r + handler.workerRequest = r if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName), zap.String("url", r.RequestURI)) + c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI)) } - if err := updateServerContext(thread, r, false, true); err != nil { + if err := updateServerContext(handler.thread, r, false, true); err != nil { // Unexpected error if c := logger.Check(zapcore.DebugLevel, "unexpected error"); c != nil { - c.Write(zap.String("worker", thread.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) + c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) } fc := r.Context().Value(contextKey).(*FrankenPHPContext) rejectRequest(fc.responseWriter, err.Error()) maybeCloseContext(fc) - thread.workerRequest = nil - thread.Unpin() + handler.workerRequest = nil + handler.thread.Unpin() - return go_frankenphp_worker_handle_request_start(threadIndex) + return handler.waitForWorkerRequest() } return true } // return true if the worker should continue to run -func (thread *phpWorkerThread) afterScriptExecution() bool { - tearDownWorkerScript(thread, thread.worker) - currentState := w.state.get() +func (handler *phpWorkerThread) afterScriptExecution(exitStatus int) bool { + tearDownWorkerScript(handler, exitStatus) + currentState := handler.state.get() switch currentState { case stateDrain: - thread.requestChan = make(chan *http.Request) + handler.thread.requestChan = make(chan *http.Request) return true - } case stateShuttingDown: return false } return true } -func (thread *phpWorkerThread) onShutdown(){ - state.set(stateDone) +func (handler *phpWorkerThread) onShutdown(){ + handler.state.set(stateDone) } -func setUpWorkerScript(thread *phpThread, worker *worker) { - thread.worker = worker - // if we are restarting due to file watching, set the state back to ready - if thread.state.is(stateRestarting) { - thread.state.set(stateReady) - } - - thread.backoff.reset() +func setUpWorkerScript(handler *phpWorkerThread, worker *worker) { + handler.backoff.reset() metrics.StartWorker(worker.fileName) // Create a dummy request to set up the worker @@ -153,27 +159,28 @@ func setUpWorkerScript(thread *phpThread, worker *worker) { panic(err) } - if err := updateServerContext(thread, r, true, false); err != nil { + if err := updateServerContext(handler.thread, r, true, false); err != nil { panic(err) } - thread.mainRequest = r + handler.mainRequest = r if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { - c.Write(zap.String("worker", worker.fileName), zap.Int("thread", thread.threadIndex)) + c.Write(zap.String("worker", worker.fileName), zap.Int("thread", handler.thread.threadIndex)) } } -func tearDownWorkerScript(thread *phpThread, exitStatus int) { - fc := thread.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) +func tearDownWorkerScript(handler *phpWorkerThread, exitStatus int) { + fc := handler.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.exitStatus = exitStatus defer func() { maybeCloseContext(fc) - thread.mainRequest = nil + handler.mainRequest = nil + handler.workerRequest = nil }() // on exit status 0 we just run the worker script again - worker := thread.worker + worker := handler.worker if fc.exitStatus == 0 { // TODO: make the max restart configurable metrics.StopWorker(worker.fileName, StopReasonRestart) @@ -184,9 +191,11 @@ func tearDownWorkerScript(thread *phpThread, exitStatus int) { return } + // TODO: error status + // on exit status 1 we apply an exponential backoff when restarting metrics.StopWorker(worker.fileName, StopReasonCrash) - thread.backoff.trigger(func(failureCount int) { + handler.backoff.trigger(func(failureCount int) { // if we end up here, the worker has not been up for backoff*2 // this is probably due to a syntax error or another fatal error if !watcherIsEnabled { @@ -196,9 +205,36 @@ func tearDownWorkerScript(thread *phpThread, exitStatus int) { }) } -func (thread *phpWorkerThread) getKnownVariableKeys() map[string]*C.zend_string{ - return thread.knownVariableKeys +//export go_frankenphp_worker_handle_request_start +func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { + handler := phpThreads[threadIndex].handler.(*phpWorkerThread) + return C.bool(handler.waitForWorkerRequest()) } -func (thread *phpWorkerThread) setKnownVariableKeys(map[string]*C.zend_string){ - thread.knownVariableKeys = knownVariableKeys + +//export go_frankenphp_finish_worker_request +func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { + thread := phpThreads[threadIndex] + r := thread.getActiveRequest() + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + + maybeCloseContext(fc) + thread.handler.(*phpWorkerThread).workerRequest = nil + thread.Unpin() + + if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { + c.Write(zap.String("worker", fc.scriptFilename), zap.String("url", r.RequestURI)) + } +} + +// when frankenphp_finish_request() is directly called from PHP +// +//export go_frankenphp_finish_php_request +func go_frankenphp_finish_php_request(threadIndex C.uintptr_t) { + r := phpThreads[threadIndex].getActiveRequest() + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + maybeCloseContext(fc) + + if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { + c.Write(zap.String("url", r.RequestURI)) + } } \ No newline at end of file diff --git a/thread_state.go b/thread_state.go index f26f7d653..c15cbb0a2 100644 --- a/thread_state.go +++ b/thread_state.go @@ -16,9 +16,11 @@ const ( stateShuttingDown stateDone stateRestarting + stateDrain + stateYielding ) -type threadStateHandler struct { +type stateHandler struct { currentState threadState mu sync.RWMutex subscribers []stateSubscriber @@ -31,19 +33,27 @@ type stateSubscriber struct { yieldFor *sync.WaitGroup } -func (h *threadStateHandler) is(state threadState) bool { +func newStateHandler() *stateHandler { + return &stateHandler{ + currentState: stateBooting, + subscribers: []stateSubscriber{}, + mu: sync.RWMutex{}, + } +} + +func (h *stateHandler) is(state threadState) bool { h.mu.RLock() defer h.mu.RUnlock() return h.currentState == state } -func (h *threadStateHandler) get(state threadState) threadState { +func (h *stateHandler) get() threadState { h.mu.RLock() defer h.mu.RUnlock() return h.currentState } -func (h *threadStateHandler) set(nextState threadState) { +func (h *stateHandler) set(nextState threadState) { h.mu.Lock() defer h.mu.Unlock() if h.currentState == nextState { @@ -76,18 +86,18 @@ func (h *threadStateHandler) set(nextState threadState) { } // wait for the thread to reach a certain state -func (h *threadStateHandler) waitFor(states ...threadState) { +func (h *stateHandler) waitFor(states ...threadState) { h.waitForStates(states, nil) } // make the thread yield to a WaitGroup once it reaches the state // this makes sure all threads are in sync both ways -func (h *threadStateHandler) waitForAndYield(yieldFor *sync.WaitGroup, states ...threadState) { +func (h *stateHandler) waitForAndYield(yieldFor *sync.WaitGroup, states ...threadState) { h.waitForStates(states, yieldFor) } // subscribe to a state and wait until the thread reaches it -func (h *threadStateHandler) waitForStates(states []threadState, yieldFor *sync.WaitGroup) { +func (h *stateHandler) waitForStates(states []threadState, yieldFor *sync.WaitGroup) { h.mu.Lock() if slices.Contains(states, h.currentState) { h.mu.Unlock() diff --git a/thread_state_test.go b/thread_state_test.go index d9ff5fcdb..693e1e3ba 100644 --- a/thread_state_test.go +++ b/thread_state_test.go @@ -9,7 +9,7 @@ import ( ) func TestYieldToEachOtherViaThreadStates(t *testing.T) { - threadState := &threadStateHandler{currentState: stateBooting} + threadState := &stateHandler{currentState: stateBooting} go func() { threadState.waitFor(stateInactive) @@ -24,7 +24,7 @@ func TestYieldToEachOtherViaThreadStates(t *testing.T) { func TestYieldToAWaitGroupPassedByThreadState(t *testing.T) { logger, _ = zap.NewDevelopment() - threadState := &threadStateHandler{currentState: stateBooting} + threadState := &stateHandler{currentState: stateBooting} hasYielded := false wg := sync.WaitGroup{} wg.Add(1) diff --git a/worker.go b/worker.go index 35c862298..b7757acb3 100644 --- a/worker.go +++ b/worker.go @@ -6,13 +6,10 @@ import ( "fmt" "github.com/dunglas/frankenphp/internal/fastabs" "net/http" - "path/filepath" "sync" "time" "github.com/dunglas/frankenphp/internal/watcher" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" ) type worker struct { @@ -86,8 +83,6 @@ func drainWorkers() { } func restartWorkers() { - restart := sync.WaitGroup{} - restart.Add(1) ready := sync.WaitGroup{} for _, worker := range workers { worker.threadMutex.RLock() @@ -95,7 +90,7 @@ func restartWorkers() { for _, thread := range worker.threads { thread.state.set(stateRestarting) go func(thread *phpThread) { - thread.state.waitForAndYield(&restart, stateReady) + thread.state.waitFor(stateYielding) ready.Done() }(thread) } @@ -103,8 +98,12 @@ func restartWorkers() { } stopWorkers() ready.Wait() + for _, worker := range workers { + for _, thread := range worker.threads { + thread.state.set(stateReady) + } + } workersDone = make(chan interface{}) - restart.Done() } func getDirectoriesToWatch(workerOpts []workerOpt) []string { @@ -116,32 +115,8 @@ func getDirectoriesToWatch(workerOpts []workerOpt) []string { } func (worker *worker) startNewThread() { - getInactivePHPThread().setActive( - // onStartup => right before the thread is ready - func(thread *phpThread) { - thread.worker = worker - thread.scriptName = worker.fileName - thread.requestChan = make(chan *http.Request) - thread.backoff = newExponentialBackoff() - worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() - metrics.ReadyWorker(worker.fileName) - }, - // beforeScriptExecution => set up the worker with a fake request - func(thread *phpThread) { - worker.beforeScript(thread) - }, - // afterScriptExecution => tear down the worker - func(thread *phpThread, exitStatus int) { - worker.afterScript(thread, exitStatus) - }, - // onShutdown => after the thread is done - func(thread *phpThread) { - thread.worker = nil - thread.backoff = nil - }, - ) + thread := getInactivePHPThread() + convertToWorkerThread(thread, worker) } func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { @@ -168,36 +143,3 @@ func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { metrics.StopWorkerRequest(worker.fileName, time.Since(fc.startedAt)) } -//export go_frankenphp_worker_handle_request_start -func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { - thread := phpWorkerThread(phpThreads[threadIndex]) - return C.bool(thread.stateMachine.waitForWorkerRequest(stateReady)) -} - -//export go_frankenphp_finish_worker_request -func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { - thread := phpThreads[threadIndex] - r := thread.getActiveRequest() - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - - maybeCloseContext(fc) - thread.workerRequest = nil - thread.Unpin() - - if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { - c.Write(zap.String("worker", fc.scriptFilename), zap.String("url", r.RequestURI)) - } -} - -// when frankenphp_finish_request() is directly called from PHP -// -//export go_frankenphp_finish_php_request -func go_frankenphp_finish_php_request(threadIndex C.uintptr_t) { - r := phpThreads[threadIndex].getActiveRequest() - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - maybeCloseContext(fc) - - if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { - c.Write(zap.String("url", r.RequestURI)) - } -} From 6747d15616118a03e705146112ba49ceae62c529 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Fri, 6 Dec 2024 23:54:23 +0100 Subject: [PATCH 39/64] Fixes tests. --- frankenphp.c | 14 ++-- php_regular_thread.go | 23 +++++-- php_thread.go | 6 +- php_thread_test.go | 28 -------- php_threads.go | 4 +- php_threads_test.go | 155 ------------------------------------------ php_worker_thread.go | 30 +++++--- worker.go | 3 + 8 files changed, 55 insertions(+), 208 deletions(-) delete mode 100644 php_thread_test.go diff --git a/frankenphp.c b/frankenphp.c index c033366b6..f2c50dd71 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -834,15 +834,15 @@ static void *php_thread(void *arg) { while (true) { char *scriptName = go_frankenphp_before_script_execution(thread_index); - // if the script name is NULL, the thread should exit - if (scriptName == NULL) { - break; - } - + int exit_status = 0; // if the script name is not empty, execute the PHP script if (strlen(scriptName) != 0) { - int exit_status = frankenphp_execute_script(scriptName); - go_frankenphp_after_script_execution(thread_index, exit_status); + exit_status = frankenphp_execute_script(scriptName); + } + + // if go signals to stop, break the loop + if(!go_frankenphp_after_script_execution(thread_index, exit_status)){ + break; } } diff --git a/php_regular_thread.go b/php_regular_thread.go index 001f5304f..e97583585 100644 --- a/php_regular_thread.go +++ b/php_regular_thread.go @@ -4,6 +4,7 @@ package frankenphp import "C" import ( "net/http" + "go.uber.org/zap" ) type phpRegularThread struct { @@ -39,11 +40,14 @@ func (thread *phpRegularThread) beforeScriptExecution() string { currentState := thread.state.get() switch currentState { case stateInactive: + logger.Info("waiting for activation", zap.Int("threadIndex", thread.thread.threadIndex),zap.Int("state", int(thread.state.get()))) thread.state.waitFor(stateActive, stateShuttingDown) + logger.Info("activated", zap.Int("threadIndex", thread.thread.threadIndex),zap.Int("state", int(thread.state.get()))) return thread.beforeScriptExecution() case stateShuttingDown: return "" case stateReady, stateActive: + logger.Info("beforeScriptExecution", zap.Int("state", int(thread.state.get()))) return waitForScriptExecution(thread) } return "" @@ -67,20 +71,21 @@ func (thread *phpRegularThread) onShutdown(){ thread.state.set(stateDone) } -func waitForScriptExecution(thread *phpRegularThread) string { +func waitForScriptExecution(handler *phpRegularThread) string { select { - case <-done: + case <-handler.thread.drainChan: + logger.Info("drainChan", zap.Int("threadIndex", handler.thread.threadIndex)) // no script should be executed if the server is shutting down return "" case r := <-requestChan: - thread.activeRequest = r + handler.activeRequest = r fc := r.Context().Value(contextKey).(*FrankenPHPContext) - if err := updateServerContext(thread.thread, r, true, false); err != nil { + if err := updateServerContext(handler.thread, r, true, false); err != nil { rejectRequest(fc.responseWriter, err.Error()) - thread.afterRequest(0) - thread.thread.Unpin() + handler.afterRequest(0) + handler.thread.Unpin() // no script should be executed if the request was rejected return "" } @@ -91,6 +96,12 @@ func waitForScriptExecution(thread *phpRegularThread) string { } func (thread *phpRegularThread) afterRequest(exitStatus int) { + + // if the request is nil, no script was executed + if thread.activeRequest == nil { + return + } + fc := thread.activeRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.exitStatus = exitStatus maybeCloseContext(fc) diff --git a/php_thread.go b/php_thread.go index 4d1fd5b3a..39f09fafe 100644 --- a/php_thread.go +++ b/php_thread.go @@ -16,6 +16,7 @@ type phpThread struct { threadIndex int knownVariableKeys map[string]*C.zend_string requestChan chan *http.Request + drainChan chan struct{} handler threadHandler state *stateHandler } @@ -66,13 +67,14 @@ func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { } //export go_frankenphp_after_script_execution -func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C.int) { +func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C.int) C.bool { thread := phpThreads[threadIndex] if exitStatus < 0 { panic(ScriptExecutionError) } - thread.handler.afterScriptExecution(int(exitStatus)) + shouldContinueExecution := thread.handler.afterScriptExecution(int(exitStatus)) thread.Unpin() + return C.bool(shouldContinueExecution) } //export go_frankenphp_on_thread_shutdown diff --git a/php_thread_test.go b/php_thread_test.go deleted file mode 100644 index eba873d5b..000000000 --- a/php_thread_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package frankenphp - -import ( - "net/http" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMainRequestIsActiveRequest(t *testing.T) { - mainRequest := &http.Request{} - thread := phpThread{} - - thread.mainRequest = mainRequest - - assert.Equal(t, mainRequest, thread.getActiveRequest()) -} - -func TestWorkerRequestIsActiveRequest(t *testing.T) { - mainRequest := &http.Request{} - workerRequest := &http.Request{} - thread := phpThread{} - - thread.mainRequest = mainRequest - thread.workerRequest = workerRequest - - assert.Equal(t, workerRequest, thread.getActiveRequest()) -} diff --git a/php_threads.go b/php_threads.go index 67486a259..ecd69f7d5 100644 --- a/php_threads.go +++ b/php_threads.go @@ -20,7 +20,8 @@ func initPHPThreads(numThreads int) error { for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{ threadIndex: i, - state: newStateHandler(), + drainChan: make(chan struct{}), + state: newStateHandler(), } convertToInactiveThread(phpThreads[i]) } @@ -52,6 +53,7 @@ func drainPHPThreads() { doneWG.Add(len(phpThreads)) for _, thread := range phpThreads { thread.state.set(stateShuttingDown) + close(thread.drainChan) } close(done) for _, thread := range phpThreads { diff --git a/php_threads_test.go b/php_threads_test.go index ab85c783f..74aa75145 100644 --- a/php_threads_test.go +++ b/php_threads_test.go @@ -1,10 +1,6 @@ package frankenphp import ( - "net/http" - "path/filepath" - "sync" - "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -18,158 +14,7 @@ func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { assert.Len(t, phpThreads, 1) assert.Equal(t, 0, phpThreads[0].threadIndex) assert.True(t, phpThreads[0].state.is(stateInactive)) - assert.Nil(t, phpThreads[0].worker) drainPHPThreads() assert.Nil(t, phpThreads) } - -// We'll start 100 threads and check that their hooks work correctly -func TestStartAndStop100PHPThreadsThatDoNothing(t *testing.T) { - logger = zap.NewNop() // the logger needs to not be nil - numThreads := 100 - readyThreads := atomic.Uint64{} - finishedThreads := atomic.Uint64{} - workingThreads := atomic.Uint64{} - workWG := sync.WaitGroup{} - workWG.Add(numThreads) - - assert.NoError(t, initPHPThreads(numThreads)) - - for i := 0; i < numThreads; i++ { - newThread := getInactivePHPThread() - newThread.setActive( - // onStartup => before the thread is ready - func(thread *phpThread) { - if thread.threadIndex == newThread.threadIndex { - readyThreads.Add(1) - } - }, - // beforeScriptExecution => we stop here immediately - func(thread *phpThread) { - if thread.threadIndex == newThread.threadIndex { - workingThreads.Add(1) - } - workWG.Done() - newThread.setInactive() - }, - // afterScriptExecution => no script is executed, we shouldn't reach here - func(thread *phpThread, exitStatus int) { - panic("hook afterScriptExecution should not be called here") - }, - // onShutdown => after the thread is done - func(thread *phpThread) { - if thread.threadIndex == newThread.threadIndex { - finishedThreads.Add(1) - } - }, - ) - } - - workWG.Wait() - drainPHPThreads() - - assert.Equal(t, numThreads, int(readyThreads.Load())) - assert.Equal(t, numThreads, int(workingThreads.Load())) - assert.Equal(t, numThreads, int(finishedThreads.Load())) -} - -// This test calls sleep() 10.000 times for 1ms in 100 PHP threads. -func TestSleep10000TimesIn100Threads(t *testing.T) { - logger, _ = zap.NewDevelopment() // the logger needs to not be nil - numThreads := 100 - maxExecutions := 10000 - executionMutex := sync.Mutex{} - executionCount := 0 - scriptPath, _ := filepath.Abs("./testdata/sleep.php") - workWG := sync.WaitGroup{} - workWG.Add(maxExecutions) - - assert.NoError(t, initPHPThreads(numThreads)) - - for i := 0; i < numThreads; i++ { - getInactivePHPThread().setActive( - // onStartup => fake a request on startup (like a worker would do) - func(thread *phpThread) { - r, _ := http.NewRequest(http.MethodGet, "sleep.php", nil) - r, _ = NewRequestWithContext(r, WithRequestDocumentRoot("/", false)) - assert.NoError(t, updateServerContext(thread, r, true, false)) - thread.mainRequest = r - thread.scriptName = scriptPath - }, - // beforeScriptExecution => execute the sleep.php script until we reach maxExecutions - func(thread *phpThread) { - executionMutex.Lock() - if executionCount >= maxExecutions { - executionMutex.Unlock() - thread.setInactive() - return - } - executionCount++ - workWG.Done() - executionMutex.Unlock() - }, - // afterScriptExecution => check the exit status of the script - func(thread *phpThread, exitStatus int) { - if int(exitStatus) != 0 { - panic("script execution failed: " + scriptPath) - } - }, - // onShutdown => nothing to do here - nil, - ) - } - - workWG.Wait() - drainPHPThreads() - - assert.Equal(t, maxExecutions, executionCount) -} - -// TODO: Make this test more chaotic -func TestStart100ThreadsAndConvertThemToDifferentThreads10Times(t *testing.T) { - logger = zap.NewNop() // the logger needs to not be nil - numThreads := 100 - numConversions := 10 - startUpTypes := make([]atomic.Uint64, numConversions) - workTypes := make([]atomic.Uint64, numConversions) - shutdownTypes := make([]atomic.Uint64, numConversions) - workWG := sync.WaitGroup{} - - assert.NoError(t, initPHPThreads(numThreads)) - - for i := 0; i < numConversions; i++ { - workWG.Add(numThreads) - numberOfConversion := i - for j := 0; j < numThreads; j++ { - getInactivePHPThread().setActive( - // onStartup => before the thread is ready - func(thread *phpThread) { - startUpTypes[numberOfConversion].Add(1) - }, - // beforeScriptExecution => while the thread is running - func(thread *phpThread) { - workTypes[numberOfConversion].Add(1) - thread.setInactive() - workWG.Done() - }, - // afterScriptExecution => we don't execute a script - nil, - // onShutdown => after the thread is done - func(thread *phpThread) { - shutdownTypes[numberOfConversion].Add(1) - }, - ) - } - workWG.Wait() - } - - drainPHPThreads() - - // each type of thread needs to have started, worked and stopped the same amount of times - for i := 0; i < numConversions; i++ { - assert.Equal(t, numThreads, int(startUpTypes[i].Load())) - assert.Equal(t, numThreads, int(workTypes[i].Load())) - assert.Equal(t, numThreads, int(shutdownTypes[i].Load())) - } -} diff --git a/php_worker_thread.go b/php_worker_thread.go index 0a87f3de0..7226beba0 100644 --- a/php_worker_thread.go +++ b/php_worker_thread.go @@ -15,7 +15,7 @@ type phpWorkerThread struct { state *stateHandler thread *phpThread worker *worker - mainRequest *http.Request + fakeRequest *http.Request workerRequest *http.Request backoff *exponentialBackoff } @@ -44,7 +44,7 @@ func (handler *phpWorkerThread) getActiveRequest() *http.Request { return handler.workerRequest } - return handler.mainRequest + return handler.fakeRequest } func (t *phpWorkerThread) isReadyToTransition() bool { @@ -78,14 +78,14 @@ func (handler *phpWorkerThread) waitForWorkerRequest() bool { c.Write(zap.String("worker", handler.worker.fileName)) } - if !handler.state.is(stateReady) { + if handler.state.is(stateActive) { metrics.ReadyWorker(handler.worker.fileName) handler.state.set(stateReady) } var r *http.Request select { - case <-workersDone: + case <-handler.thread.drainChan: if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { c.Write(zap.String("worker", handler.worker.fileName)) } @@ -163,20 +163,32 @@ func setUpWorkerScript(handler *phpWorkerThread, worker *worker) { panic(err) } - handler.mainRequest = r + handler.fakeRequest = r if c := logger.Check(zapcore.DebugLevel, "starting"); c != nil { c.Write(zap.String("worker", worker.fileName), zap.Int("thread", handler.thread.threadIndex)) } } func tearDownWorkerScript(handler *phpWorkerThread, exitStatus int) { - fc := handler.mainRequest.Context().Value(contextKey).(*FrankenPHPContext) - fc.exitStatus = exitStatus - defer func() { + // if the fake request is nil, no script was executed + if handler.fakeRequest == nil { + return + } + + // if the worker request is not nil, the script might have crashed + // make sure to close the worker request context + if handler.workerRequest != nil { + fc := handler.workerRequest.Context().Value(contextKey).(*FrankenPHPContext) maybeCloseContext(fc) - handler.mainRequest = nil handler.workerRequest = nil + } + + fc := handler.fakeRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + + defer func() { + handler.fakeRequest = nil }() // on exit status 0 we just run the worker script again diff --git a/worker.go b/worker.go index b7757acb3..a533624d3 100644 --- a/worker.go +++ b/worker.go @@ -12,6 +12,7 @@ import ( "github.com/dunglas/frankenphp/internal/watcher" ) +// represents a worker script and can have many threads assigned to it type worker struct { fileName string num int @@ -89,6 +90,7 @@ func restartWorkers() { ready.Add(len(worker.threads)) for _, thread := range worker.threads { thread.state.set(stateRestarting) + close(thread.drainChan) go func(thread *phpThread) { thread.state.waitFor(stateYielding) ready.Done() @@ -100,6 +102,7 @@ func restartWorkers() { ready.Wait() for _, worker := range workers { for _, thread := range worker.threads { + thread.drainChan = make(chan struct{}) thread.state.set(stateReady) } } From 54dc2675baddd3f4e3e48cb99257d0006b918524 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 10:58:51 +0100 Subject: [PATCH 40/64] Refactoring. --- frankenphp.c | 2 - inactive-thread.go | 47 ++++++++++ php_inactive_thread.go | 49 ---------- php_regular_thread.go | 109 ---------------------- php_thread.go | 10 +- php_threads.go | 6 +- regular-thread.go | 101 ++++++++++++++++++++ thread-state.go | 90 ++++++++++++++++++ thread-state_test.go | 22 +++++ thread_state.go | 114 ----------------------- thread_state_test.go | 43 --------- php_worker_thread.go => worker-thread.go | 46 ++++----- 12 files changed, 288 insertions(+), 351 deletions(-) create mode 100644 inactive-thread.go delete mode 100644 php_inactive_thread.go delete mode 100644 php_regular_thread.go create mode 100644 regular-thread.go create mode 100644 thread-state.go create mode 100644 thread-state_test.go delete mode 100644 thread_state.go delete mode 100644 thread_state_test.go rename php_worker_thread.go => worker-thread.go (85%) diff --git a/frankenphp.c b/frankenphp.c index f2c50dd71..40002a534 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -828,8 +828,6 @@ static void *php_thread(void *arg) { cfg_get_string("filter.default", &default_filter); should_filter_var = default_filter != NULL; - go_frankenphp_on_thread_startup(thread_index); - // perform work until go signals to stop while (true) { char *scriptName = go_frankenphp_before_script_execution(thread_index); diff --git a/inactive-thread.go b/inactive-thread.go new file mode 100644 index 000000000..cfb589706 --- /dev/null +++ b/inactive-thread.go @@ -0,0 +1,47 @@ +package frankenphp + +import ( + "net/http" + "strconv" +) + +// representation of a thread with no work assigned to it +// implements the threadHandler interface +type inactiveThread struct { + state *threadState +} + +func convertToInactiveThread(thread *phpThread) { + thread.handler = &inactiveThread{state: thread.state} +} + +func (t *inactiveThread) isReadyToTransition() bool { + return true +} + +func (thread *inactiveThread) getActiveRequest() *http.Request { + panic("idle threads have no requests") +} + +func (thread *inactiveThread) beforeScriptExecution() string { + // no script execution for inactive threads + return "" +} + +func (thread *inactiveThread) afterScriptExecution(exitStatus int) bool { + thread.state.set(stateInactive) + // wait for external signal to start or shut down + thread.state.waitFor(stateActive, stateShuttingDown) + switch thread.state.get() { + case stateActive: + return true + case stateShuttingDown: + return false + } + panic("unexpected state: "+strconv.Itoa(int(thread.state.get()))) +} + +func (thread *inactiveThread) onShutdown(){ + thread.state.set(stateDone) +} + diff --git a/php_inactive_thread.go b/php_inactive_thread.go deleted file mode 100644 index ff816f7b9..000000000 --- a/php_inactive_thread.go +++ /dev/null @@ -1,49 +0,0 @@ -package frankenphp - -import ( - "net/http" - "strconv" -) - -type phpInactiveThread struct { - state *stateHandler -} - -func convertToInactiveThread(thread *phpThread) { - thread.handler = &phpInactiveThread{state: thread.state} -} - -func (t *phpInactiveThread) isReadyToTransition() bool { - return true -} - -// this is done once -func (thread *phpInactiveThread) onStartup(){ - // do nothing -} - -func (thread *phpInactiveThread) getActiveRequest() *http.Request { - panic("idle threads have no requests") -} - -func (thread *phpInactiveThread) beforeScriptExecution() string { - thread.state.set(stateInactive) - return "" -} - -func (thread *phpInactiveThread) afterScriptExecution(exitStatus int) bool { - // wait for external signal to start or shut down - thread.state.waitFor(stateActive, stateShuttingDown) - switch thread.state.get() { - case stateActive: - return true - case stateShuttingDown: - return false - } - panic("unexpected state: "+strconv.Itoa(int(thread.state.get()))) -} - -func (thread *phpInactiveThread) onShutdown(){ - thread.state.set(stateDone) -} - diff --git a/php_regular_thread.go b/php_regular_thread.go deleted file mode 100644 index e97583585..000000000 --- a/php_regular_thread.go +++ /dev/null @@ -1,109 +0,0 @@ -package frankenphp - -// #include "frankenphp.h" -import "C" -import ( - "net/http" - "go.uber.org/zap" -) - -type phpRegularThread struct { - state *stateHandler - thread *phpThread - activeRequest *http.Request -} - -func convertToRegularThread(thread *phpThread) { - thread.handler = &phpRegularThread{ - thread: thread, - state: thread.state, - } - thread.handler.onStartup() - thread.state.set(stateActive) -} - -func (t *phpRegularThread) isReadyToTransition() bool { - return false -} - -// this is done once -func (thread *phpRegularThread) onStartup(){ - // do nothing -} - -func (thread *phpRegularThread) getActiveRequest() *http.Request { - return thread.activeRequest -} - -// return the name of the script or an empty string if no script should be executed -func (thread *phpRegularThread) beforeScriptExecution() string { - currentState := thread.state.get() - switch currentState { - case stateInactive: - logger.Info("waiting for activation", zap.Int("threadIndex", thread.thread.threadIndex),zap.Int("state", int(thread.state.get()))) - thread.state.waitFor(stateActive, stateShuttingDown) - logger.Info("activated", zap.Int("threadIndex", thread.thread.threadIndex),zap.Int("state", int(thread.state.get()))) - return thread.beforeScriptExecution() - case stateShuttingDown: - return "" - case stateReady, stateActive: - logger.Info("beforeScriptExecution", zap.Int("state", int(thread.state.get()))) - return waitForScriptExecution(thread) - } - return "" -} - -// return true if the worker should continue to run -func (thread *phpRegularThread) afterScriptExecution(exitStatus int) bool { - thread.afterRequest(exitStatus) - - currentState := thread.state.get() - switch currentState { - case stateDrain: - return true - case stateShuttingDown: - return false - } - return true -} - -func (thread *phpRegularThread) onShutdown(){ - thread.state.set(stateDone) -} - -func waitForScriptExecution(handler *phpRegularThread) string { - select { - case <-handler.thread.drainChan: - logger.Info("drainChan", zap.Int("threadIndex", handler.thread.threadIndex)) - // no script should be executed if the server is shutting down - return "" - - case r := <-requestChan: - handler.activeRequest = r - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - - if err := updateServerContext(handler.thread, r, true, false); err != nil { - rejectRequest(fc.responseWriter, err.Error()) - handler.afterRequest(0) - handler.thread.Unpin() - // no script should be executed if the request was rejected - return "" - } - - // set the scriptName that should be executed - return fc.scriptFilename - } -} - -func (thread *phpRegularThread) afterRequest(exitStatus int) { - - // if the request is nil, no script was executed - if thread.activeRequest == nil { - return - } - - fc := thread.activeRequest.Context().Value(contextKey).(*FrankenPHPContext) - fc.exitStatus = exitStatus - maybeCloseContext(fc) - thread.activeRequest = nil -} diff --git a/php_thread.go b/php_thread.go index 39f09fafe..a16e1a573 100644 --- a/php_thread.go +++ b/php_thread.go @@ -18,12 +18,11 @@ type phpThread struct { requestChan chan *http.Request drainChan chan struct{} handler threadHandler - state *stateHandler + state *threadState } // interface that defines how the callbacks from the C thread should be handled type threadHandler interface { - onStartup() beforeScriptExecution() string afterScriptExecution(exitStatus int) bool onShutdown() @@ -53,11 +52,6 @@ func (thread *phpThread) pinCString(s string) *C.char { return thread.pinString(s + "\x00") } -//export go_frankenphp_on_thread_startup -func go_frankenphp_on_thread_startup(threadIndex C.uintptr_t) { - phpThreads[threadIndex].handler.onStartup() -} - //export go_frankenphp_before_script_execution func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] @@ -79,5 +73,5 @@ func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C. //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { - phpThreads[threadIndex].handler.onShutdown() + phpThreads[threadIndex].state.set(stateDone) } diff --git a/php_threads.go b/php_threads.go index ecd69f7d5..40f787621 100644 --- a/php_threads.go +++ b/php_threads.go @@ -10,7 +10,7 @@ import ( var ( phpThreads []*phpThread done chan struct{} - mainThreadState *stateHandler + mainThreadState *threadState ) // reserve a fixed number of PHP threads on the go side @@ -21,7 +21,7 @@ func initPHPThreads(numThreads int) error { phpThreads[i] = &phpThread{ threadIndex: i, drainChan: make(chan struct{}), - state: newStateHandler(), + state: newThreadState(), } convertToInactiveThread(phpThreads[i]) } @@ -69,7 +69,7 @@ func drainPHPThreads() { } func startMainThread(numThreads int) error { - mainThreadState = newStateHandler() + mainThreadState = newThreadState() if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { return MainThreadCreationError } diff --git a/regular-thread.go b/regular-thread.go new file mode 100644 index 000000000..371ae517d --- /dev/null +++ b/regular-thread.go @@ -0,0 +1,101 @@ +package frankenphp + +// #include "frankenphp.h" +import "C" +import ( + "net/http" +) + +// representation of a non-worker PHP thread +// executes PHP scripts in a web context +// implements the threadHandler interface +type regularThread struct { + state *threadState + thread *phpThread + activeRequest *http.Request +} + +func convertToRegularThread(thread *phpThread) { + thread.handler = ®ularThread{ + thread: thread, + state: thread.state, + } + thread.state.set(stateActive) +} + +func (t *regularThread) isReadyToTransition() bool { + return false +} + +func (handler *regularThread) getActiveRequest() *http.Request { + return handler.activeRequest +} + +// return the name of the script or an empty string if no script should be executed +func (handler *regularThread) beforeScriptExecution() string { + currentState := handler.state.get() + switch currentState { + case stateInactive: + handler.state.waitFor(stateActive, stateShuttingDown) + return handler.beforeScriptExecution() + case stateShuttingDown: + return "" + case stateReady, stateActive: + return handler.waitForScriptExecution() + } + return "" +} + +// return true if the worker should continue to run +func (handler *regularThread) afterScriptExecution(exitStatus int) bool { + handler.afterRequest(exitStatus) + + currentState := handler.state.get() + switch currentState { + case stateDrain: + return true + case stateShuttingDown: + return false + } + return true +} + +func (handler *regularThread) onShutdown(){ + handler.state.set(stateDone) +} + +func (handler *regularThread) waitForScriptExecution() string { + select { + case <-handler.thread.drainChan: + // no script should be executed if the server is shutting down + return "" + + case r := <-requestChan: + handler.activeRequest = r + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + + if err := updateServerContext(handler.thread, r, true, false); err != nil { + rejectRequest(fc.responseWriter, err.Error()) + handler.afterRequest(0) + handler.thread.Unpin() + // no script should be executed if the request was rejected + return "" + } + + // set the scriptName that should be executed + return fc.scriptFilename + } +} + +func (handler *regularThread) afterRequest(exitStatus int) { + + // if the request is nil, no script was executed + if handler.activeRequest == nil { + return + } + + fc := handler.activeRequest.Context().Value(contextKey).(*FrankenPHPContext) + fc.exitStatus = exitStatus + maybeCloseContext(fc) + handler.activeRequest = nil +} diff --git a/thread-state.go b/thread-state.go new file mode 100644 index 000000000..4abf00c5e --- /dev/null +++ b/thread-state.go @@ -0,0 +1,90 @@ +package frankenphp + +import ( + "slices" + "sync" +) + +type stateID int + +const ( + stateBooting stateID = iota + stateInactive + stateActive + stateReady + stateBusy + stateShuttingDown + stateDone + stateRestarting + stateDrain + stateYielding +) + +type threadState struct { + currentState stateID + mu sync.RWMutex + subscribers []stateSubscriber +} + + +type stateSubscriber struct { + states []stateID + ch chan struct{} +} + +func newThreadState() *threadState { + return &threadState{ + currentState: stateBooting, + subscribers: []stateSubscriber{}, + mu: sync.RWMutex{}, + } +} + +func (h *threadState) is(state stateID) bool { + h.mu.RLock() + defer h.mu.RUnlock() + return h.currentState == state +} + +func (h *threadState) get() stateID { + h.mu.RLock() + defer h.mu.RUnlock() + return h.currentState +} + +func (h *threadState) set(nextState stateID) { + h.mu.Lock() + defer h.mu.Unlock() + h.currentState = nextState + + if len(h.subscribers) == 0 { + return + } + + newSubscribers := []stateSubscriber{} + // notify subscribers to the state change + for _, sub := range h.subscribers { + if !slices.Contains(sub.states, nextState) { + newSubscribers = append(newSubscribers, sub) + continue + } + close(sub.ch) + } + h.subscribers = newSubscribers +} + +// block until the thread reaches a certain state +func (h *threadState) waitFor(states ...stateID) { + h.mu.Lock() + if slices.Contains(states, h.currentState) { + h.mu.Unlock() + return + } + sub := stateSubscriber{ + states: states, + ch: make(chan struct{}), + } + h.subscribers = append(h.subscribers, sub) + h.mu.Unlock() + <-sub.ch +} diff --git a/thread-state_test.go b/thread-state_test.go new file mode 100644 index 000000000..29c68a810 --- /dev/null +++ b/thread-state_test.go @@ -0,0 +1,22 @@ +package frankenphp + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestYieldToEachOtherViaThreadStates(t *testing.T) { + threadState := &threadState{currentState: stateBooting} + + go func() { + threadState.waitFor(stateInactive) + assert.True(t, threadState.is(stateInactive)) + threadState.set(stateActive) + }() + + threadState.set(stateInactive) + threadState.waitFor(stateActive) + assert.True(t, threadState.is(stateActive)) +} + diff --git a/thread_state.go b/thread_state.go deleted file mode 100644 index c15cbb0a2..000000000 --- a/thread_state.go +++ /dev/null @@ -1,114 +0,0 @@ -package frankenphp - -import ( - "slices" - "sync" -) - -type threadState int - -const ( - stateBooting threadState = iota - stateInactive - stateActive - stateReady - stateBusy - stateShuttingDown - stateDone - stateRestarting - stateDrain - stateYielding -) - -type stateHandler struct { - currentState threadState - mu sync.RWMutex - subscribers []stateSubscriber -} - - -type stateSubscriber struct { - states []threadState - ch chan struct{} - yieldFor *sync.WaitGroup -} - -func newStateHandler() *stateHandler { - return &stateHandler{ - currentState: stateBooting, - subscribers: []stateSubscriber{}, - mu: sync.RWMutex{}, - } -} - -func (h *stateHandler) is(state threadState) bool { - h.mu.RLock() - defer h.mu.RUnlock() - return h.currentState == state -} - -func (h *stateHandler) get() threadState { - h.mu.RLock() - defer h.mu.RUnlock() - return h.currentState -} - -func (h *stateHandler) set(nextState threadState) { - h.mu.Lock() - defer h.mu.Unlock() - if h.currentState == nextState { - // TODO: do we return here or inform subscribers? - // TODO: should we ever reach here? - return - } - - h.currentState = nextState - - if len(h.subscribers) == 0 { - return - } - - newSubscribers := []stateSubscriber{} - // TODO: do we even need multiple subscribers? - // notify subscribers to the state change - for _, sub := range h.subscribers { - if !slices.Contains(sub.states, nextState) { - newSubscribers = append(newSubscribers, sub) - continue - } - close(sub.ch) - // yield for the subscriber - if sub.yieldFor != nil { - defer sub.yieldFor.Wait() - } - } - h.subscribers = newSubscribers -} - -// wait for the thread to reach a certain state -func (h *stateHandler) waitFor(states ...threadState) { - h.waitForStates(states, nil) -} - -// make the thread yield to a WaitGroup once it reaches the state -// this makes sure all threads are in sync both ways -func (h *stateHandler) waitForAndYield(yieldFor *sync.WaitGroup, states ...threadState) { - h.waitForStates(states, yieldFor) -} - -// subscribe to a state and wait until the thread reaches it -func (h *stateHandler) waitForStates(states []threadState, yieldFor *sync.WaitGroup) { - h.mu.Lock() - if slices.Contains(states, h.currentState) { - h.mu.Unlock() - return - } - sub := stateSubscriber{ - states: states, - ch: make(chan struct{}), - yieldFor: yieldFor, - } - h.subscribers = append(h.subscribers, sub) - h.mu.Unlock() - <-sub.ch -} diff --git a/thread_state_test.go b/thread_state_test.go deleted file mode 100644 index 693e1e3ba..000000000 --- a/thread_state_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package frankenphp - -import ( - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "go.uber.org/zap" -) - -func TestYieldToEachOtherViaThreadStates(t *testing.T) { - threadState := &stateHandler{currentState: stateBooting} - - go func() { - threadState.waitFor(stateInactive) - assert.True(t, threadState.is(stateInactive)) - threadState.set(stateActive) - }() - - threadState.set(stateInactive) - threadState.waitFor(stateActive) - assert.True(t, threadState.is(stateActive)) -} - -func TestYieldToAWaitGroupPassedByThreadState(t *testing.T) { - logger, _ = zap.NewDevelopment() - threadState := &stateHandler{currentState: stateBooting} - hasYielded := false - wg := sync.WaitGroup{} - wg.Add(1) - - go func() { - threadState.set(stateInactive) - threadState.waitForAndYield(&wg, stateActive) - hasYielded = true - wg.Done() - }() - - threadState.waitFor(stateInactive) - threadState.set(stateActive) - // 'set' should have yielded to the wait group - assert.True(t, hasYielded) -} diff --git a/php_worker_thread.go b/worker-thread.go similarity index 85% rename from php_worker_thread.go rename to worker-thread.go index 7226beba0..9a50b84d3 100644 --- a/php_worker_thread.go +++ b/worker-thread.go @@ -11,8 +11,11 @@ import ( "go.uber.org/zap/zapcore" ) -type phpWorkerThread struct { - state *stateHandler +// representation of a thread assigned to a worker script +// executes the PHP worker script in a loop +// implements the threadHandler interface +type workerThread struct { + state *threadState thread *phpThread worker *worker fakeRequest *http.Request @@ -21,25 +24,22 @@ type phpWorkerThread struct { } func convertToWorkerThread(thread *phpThread, worker *worker) { - thread.handler = &phpWorkerThread{ + handler := &workerThread{ state: thread.state, thread: thread, worker: worker, + backoff: newExponentialBackoff(), } - thread.handler.onStartup() - thread.state.set(stateActive) -} + thread.handler = handler + thread.requestChan = make(chan *http.Request) + worker.threadMutex.Lock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() -// this is done once -func (handler *phpWorkerThread) onStartup(){ - handler.thread.requestChan = make(chan *http.Request) - handler.backoff = newExponentialBackoff() - handler.worker.threadMutex.Lock() - handler.worker.threads = append(handler.worker.threads, handler.thread) - handler.worker.threadMutex.Unlock() + thread.state.set(stateActive) } -func (handler *phpWorkerThread) getActiveRequest() *http.Request { +func (handler *workerThread) getActiveRequest() *http.Request { if handler.workerRequest != nil { return handler.workerRequest } @@ -47,12 +47,12 @@ func (handler *phpWorkerThread) getActiveRequest() *http.Request { return handler.fakeRequest } -func (t *phpWorkerThread) isReadyToTransition() bool { +func (t *workerThread) isReadyToTransition() bool { return false } // return the name of the script or an empty string if no script should be executed -func (handler *phpWorkerThread) beforeScriptExecution() string { +func (handler *workerThread) beforeScriptExecution() string { currentState := handler.state.get() switch currentState { case stateInactive: @@ -72,7 +72,7 @@ func (handler *phpWorkerThread) beforeScriptExecution() string { return "" } -func (handler *phpWorkerThread) waitForWorkerRequest() bool { +func (handler *workerThread) waitForWorkerRequest() bool { if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { c.Write(zap.String("worker", handler.worker.fileName)) @@ -123,7 +123,7 @@ func (handler *phpWorkerThread) waitForWorkerRequest() bool { } // return true if the worker should continue to run -func (handler *phpWorkerThread) afterScriptExecution(exitStatus int) bool { +func (handler *workerThread) afterScriptExecution(exitStatus int) bool { tearDownWorkerScript(handler, exitStatus) currentState := handler.state.get() switch currentState { @@ -136,11 +136,11 @@ func (handler *phpWorkerThread) afterScriptExecution(exitStatus int) bool { return true } -func (handler *phpWorkerThread) onShutdown(){ +func (handler *workerThread) onShutdown(){ handler.state.set(stateDone) } -func setUpWorkerScript(handler *phpWorkerThread, worker *worker) { +func setUpWorkerScript(handler *workerThread, worker *worker) { handler.backoff.reset() metrics.StartWorker(worker.fileName) @@ -169,7 +169,7 @@ func setUpWorkerScript(handler *phpWorkerThread, worker *worker) { } } -func tearDownWorkerScript(handler *phpWorkerThread, exitStatus int) { +func tearDownWorkerScript(handler *workerThread, exitStatus int) { // if the fake request is nil, no script was executed if handler.fakeRequest == nil { @@ -219,7 +219,7 @@ func tearDownWorkerScript(handler *phpWorkerThread, exitStatus int) { //export go_frankenphp_worker_handle_request_start func go_frankenphp_worker_handle_request_start(threadIndex C.uintptr_t) C.bool { - handler := phpThreads[threadIndex].handler.(*phpWorkerThread) + handler := phpThreads[threadIndex].handler.(*workerThread) return C.bool(handler.waitForWorkerRequest()) } @@ -230,7 +230,7 @@ func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { fc := r.Context().Value(contextKey).(*FrankenPHPContext) maybeCloseContext(fc) - thread.handler.(*phpWorkerThread).workerRequest = nil + thread.handler.(*workerThread).workerRequest = nil thread.Unpin() if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { From 62147546562a80f00668ccf1e2966c0c0d65eb86 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 11:13:05 +0100 Subject: [PATCH 41/64] Fixes merge conflicts. --- exponential_backoff.go | 60 ------------------------------------------ worker-thread.go | 25 ++++++++++-------- worker.go | 3 +-- 3 files changed, 15 insertions(+), 73 deletions(-) delete mode 100644 exponential_backoff.go diff --git a/exponential_backoff.go b/exponential_backoff.go deleted file mode 100644 index 359e2bd4f..000000000 --- a/exponential_backoff.go +++ /dev/null @@ -1,60 +0,0 @@ -package frankenphp - -import ( - "sync" - "time" -) - -const maxBackoff = 1 * time.Second -const minBackoff = 100 * time.Millisecond -const maxConsecutiveFailures = 6 - -type exponentialBackoff struct { - backoff time.Duration - failureCount int - mu sync.RWMutex - upFunc sync.Once -} - -func newExponentialBackoff() *exponentialBackoff { - return &exponentialBackoff{backoff: minBackoff} -} - -func (e *exponentialBackoff) reset() { - e.mu.Lock() - e.upFunc = sync.Once{} - wait := e.backoff * 2 - e.mu.Unlock() - go func() { - time.Sleep(wait) - e.mu.Lock() - defer e.mu.Unlock() - e.upFunc.Do(func() { - // if we come back to a stable state, reset the failure count - if e.backoff == minBackoff { - e.failureCount = 0 - } - - // earn back the backoff over time - if e.failureCount > 0 { - e.backoff = max(e.backoff/2, minBackoff) - } - }) - }() -} - -func (e *exponentialBackoff) trigger(onMaxFailures func(failureCount int)) { - e.mu.RLock() - e.upFunc.Do(func() { - if e.failureCount >= maxConsecutiveFailures { - onMaxFailures(e.failureCount) - } - e.failureCount += 1 - }) - wait := e.backoff - e.mu.RUnlock() - time.Sleep(wait) - e.mu.Lock() - e.backoff = min(e.backoff*2, maxBackoff) - e.mu.Unlock() -} diff --git a/worker-thread.go b/worker-thread.go index 9a50b84d3..4d2907606 100644 --- a/worker-thread.go +++ b/worker-thread.go @@ -6,6 +6,7 @@ import ( "net/http" "path/filepath" "fmt" + "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -28,7 +29,11 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { state: thread.state, thread: thread, worker: worker, - backoff: newExponentialBackoff(), + backoff: &exponentialBackoff{ + maxBackoff: 1 * time.Second, + minBackoff: 100 * time.Millisecond, + maxConsecutiveFailures: 6, + }, } thread.handler = handler thread.requestChan = make(chan *http.Request) @@ -141,7 +146,7 @@ func (handler *workerThread) onShutdown(){ } func setUpWorkerScript(handler *workerThread, worker *worker) { - handler.backoff.reset() + handler.backoff.wait() metrics.StartWorker(worker.fileName) // Create a dummy request to set up the worker @@ -196,7 +201,7 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { if fc.exitStatus == 0 { // TODO: make the max restart configurable metrics.StopWorker(worker.fileName, StopReasonRestart) - + handler.backoff.recordSuccess() if c := logger.Check(zapcore.DebugLevel, "restarting"); c != nil { c.Write(zap.String("worker", worker.fileName)) } @@ -207,14 +212,12 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { // on exit status 1 we apply an exponential backoff when restarting metrics.StopWorker(worker.fileName, StopReasonCrash) - handler.backoff.trigger(func(failureCount int) { - // if we end up here, the worker has not been up for backoff*2 - // this is probably due to a syntax error or another fatal error - if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) - } - logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", failureCount)) - }) + if handler.backoff.recordFailure() { + if !watcherIsEnabled { + panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) + } + logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", handler.backoff.failureCount)) + } } //export go_frankenphp_worker_handle_request_start diff --git a/worker.go b/worker.go index 916341318..8ee25cff0 100644 --- a/worker.go +++ b/worker.go @@ -73,7 +73,6 @@ func newWorker(o workerOpt) (*worker, error) { num: o.num, env: o.env, requestChan: make(chan *http.Request), - ready: make(chan struct{}, o.num), } workers[absFileName] = w @@ -102,7 +101,6 @@ func restartWorkers() { ready.Done() }(thread) } - worker.threadMutex.RUnlock() } stopWorkers() ready.Wait() @@ -111,6 +109,7 @@ func restartWorkers() { thread.drainChan = make(chan struct{}) thread.state.set(stateReady) } + worker.threadMutex.RUnlock() } workersDone = make(chan interface{}) } From 00eb83401fedc0b4c8cdc209e1902370891e370c Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 11:27:58 +0100 Subject: [PATCH 42/64] Formatting --- frankenphp.go | 2 +- inactive-thread.go | 9 +- php_threads.go => main-thread.go | 41 +++--- php_threads_test.go => main-thread_test.go | 0 phpthread.go | 10 +- regular-thread.go | 52 +++---- thread-state.go | 29 ++-- thread-state_test.go | 1 - worker-thread.go | 150 ++++++++++----------- worker.go | 12 +- 10 files changed, 155 insertions(+), 151 deletions(-) rename php_threads.go => main-thread.go (69%) rename php_threads_test.go => main-thread_test.go (100%) diff --git a/frankenphp.go b/frankenphp.go index 10210b2f9..43ae16a7b 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -467,7 +467,7 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error metrics.StartRequest() select { - case <-done: + case <-mainThread.done: case requestChan <- request: <-fc.done } diff --git a/inactive-thread.go b/inactive-thread.go index cfb589706..80921ebcb 100644 --- a/inactive-thread.go +++ b/inactive-thread.go @@ -8,7 +8,7 @@ import ( // representation of a thread with no work assigned to it // implements the threadHandler interface type inactiveThread struct { - state *threadState + state *threadState } func convertToInactiveThread(thread *phpThread) { @@ -38,10 +38,9 @@ func (thread *inactiveThread) afterScriptExecution(exitStatus int) bool { case stateShuttingDown: return false } - panic("unexpected state: "+strconv.Itoa(int(thread.state.get()))) + panic("unexpected state: " + strconv.Itoa(int(thread.state.get()))) } -func (thread *inactiveThread) onShutdown(){ - thread.state.set(stateDone) +func (thread *inactiveThread) onShutdown() { + thread.state.set(stateDone) } - diff --git a/php_threads.go b/main-thread.go similarity index 69% rename from php_threads.go rename to main-thread.go index 40f787621..687455945 100644 --- a/php_threads.go +++ b/main-thread.go @@ -7,16 +7,26 @@ import ( "sync" ) +type mainPHPThread struct { + state *threadState + done chan struct{} + numThreads int +} + var ( - phpThreads []*phpThread - done chan struct{} - mainThreadState *threadState + phpThreads []*phpThread + mainThread *mainPHPThread ) // reserve a fixed number of PHP threads on the go side func initPHPThreads(numThreads int) error { - done = make(chan struct{}) + mainThread = &mainPHPThread{ + state: newThreadState(), + done: make(chan struct{}), + numThreads: numThreads, + } phpThreads = make([]*phpThread, numThreads) + for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{ threadIndex: i, @@ -25,13 +35,13 @@ func initPHPThreads(numThreads int) error { } convertToInactiveThread(phpThreads[i]) } - if err := startMainThread(numThreads); err != nil { + if err := mainThread.start(); err != nil { return err } // initialize all threads as inactive ready := sync.WaitGroup{} - ready.Add(len(phpThreads)) + ready.Add(numThreads) for _, thread := range phpThreads { go func() { @@ -55,7 +65,7 @@ func drainPHPThreads() { thread.state.set(stateShuttingDown) close(thread.drainChan) } - close(done) + close(mainThread.done) for _, thread := range phpThreads { go func(thread *phpThread) { thread.state.waitFor(stateDone) @@ -63,17 +73,16 @@ func drainPHPThreads() { }(thread) } doneWG.Wait() - mainThreadState.set(stateShuttingDown) - mainThreadState.waitFor(stateDone) + mainThread.state.set(stateShuttingDown) + mainThread.state.waitFor(stateDone) phpThreads = nil } -func startMainThread(numThreads int) error { - mainThreadState = newThreadState() - if C.frankenphp_new_main_thread(C.int(numThreads)) != 0 { +func (mainThread *mainPHPThread) start() error { + if C.frankenphp_new_main_thread(C.int(mainThread.numThreads)) != 0 { return MainThreadCreationError } - mainThreadState.waitFor(stateActive) + mainThread.state.waitFor(stateActive) return nil } @@ -88,11 +97,11 @@ func getInactivePHPThread() *phpThread { //export go_frankenphp_main_thread_is_ready func go_frankenphp_main_thread_is_ready() { - mainThreadState.set(stateActive) - mainThreadState.waitFor(stateShuttingDown) + mainThread.state.set(stateActive) + mainThread.state.waitFor(stateShuttingDown) } //export go_frankenphp_shutdown_main_thread func go_frankenphp_shutdown_main_thread() { - mainThreadState.set(stateDone) + mainThread.state.set(stateDone) } diff --git a/php_threads_test.go b/main-thread_test.go similarity index 100% rename from php_threads_test.go rename to main-thread_test.go diff --git a/phpthread.go b/phpthread.go index a16e1a573..465a1eb17 100644 --- a/phpthread.go +++ b/phpthread.go @@ -13,12 +13,12 @@ import ( type phpThread struct { runtime.Pinner - threadIndex int + threadIndex int knownVariableKeys map[string]*C.zend_string - requestChan chan *http.Request - drainChan chan struct{} - handler threadHandler - state *threadState + requestChan chan *http.Request + drainChan chan struct{} + handler threadHandler + state *threadState } // interface that defines how the callbacks from the C thread should be handled diff --git a/regular-thread.go b/regular-thread.go index 371ae517d..ef82c568d 100644 --- a/regular-thread.go +++ b/regular-thread.go @@ -10,15 +10,15 @@ import ( // executes PHP scripts in a web context // implements the threadHandler interface type regularThread struct { - state *threadState - thread *phpThread + state *threadState + thread *phpThread activeRequest *http.Request } func convertToRegularThread(thread *phpThread) { thread.handler = ®ularThread{ thread: thread, - state: thread.state, + state: thread.state, } thread.state.set(stateActive) } @@ -40,7 +40,7 @@ func (handler *regularThread) beforeScriptExecution() string { return handler.beforeScriptExecution() case stateShuttingDown: return "" - case stateReady, stateActive: + case stateReady, stateActive: return handler.waitForScriptExecution() } return "" @@ -53,38 +53,38 @@ func (handler *regularThread) afterScriptExecution(exitStatus int) bool { currentState := handler.state.get() switch currentState { case stateDrain: - return true + return true case stateShuttingDown: return false } return true } -func (handler *regularThread) onShutdown(){ - handler.state.set(stateDone) +func (handler *regularThread) onShutdown() { + handler.state.set(stateDone) } func (handler *regularThread) waitForScriptExecution() string { select { - case <-handler.thread.drainChan: - // no script should be executed if the server is shutting down - return "" - - case r := <-requestChan: - handler.activeRequest = r - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - - if err := updateServerContext(handler.thread, r, true, false); err != nil { - rejectRequest(fc.responseWriter, err.Error()) - handler.afterRequest(0) - handler.thread.Unpin() - // no script should be executed if the request was rejected - return "" - } - - // set the scriptName that should be executed - return fc.scriptFilename - } + case <-handler.thread.drainChan: + // no script should be executed if the server is shutting down + return "" + + case r := <-requestChan: + handler.activeRequest = r + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + + if err := updateServerContext(handler.thread, r, true, false); err != nil { + rejectRequest(fc.responseWriter, err.Error()) + handler.afterRequest(0) + handler.thread.Unpin() + // no script should be executed if the request was rejected + return "" + } + + // set the scriptName that should be executed + return fc.scriptFilename + } } func (handler *regularThread) afterRequest(exitStatus int) { diff --git a/thread-state.go b/thread-state.go index 4abf00c5e..5ca9443dd 100644 --- a/thread-state.go +++ b/thread-state.go @@ -26,17 +26,16 @@ type threadState struct { subscribers []stateSubscriber } - type stateSubscriber struct { - states []stateID - ch chan struct{} + states []stateID + ch chan struct{} } func newThreadState() *threadState { return &threadState{ currentState: stateBooting, subscribers: []stateSubscriber{}, - mu: sync.RWMutex{}, + mu: sync.RWMutex{}, } } @@ -76,15 +75,15 @@ func (h *threadState) set(nextState stateID) { // block until the thread reaches a certain state func (h *threadState) waitFor(states ...stateID) { h.mu.Lock() - if slices.Contains(states, h.currentState) { - h.mu.Unlock() - return - } - sub := stateSubscriber{ - states: states, - ch: make(chan struct{}), - } - h.subscribers = append(h.subscribers, sub) - h.mu.Unlock() - <-sub.ch + if slices.Contains(states, h.currentState) { + h.mu.Unlock() + return + } + sub := stateSubscriber{ + states: states, + ch: make(chan struct{}), + } + h.subscribers = append(h.subscribers, sub) + h.mu.Unlock() + <-sub.ch } diff --git a/thread-state_test.go b/thread-state_test.go index 29c68a810..f71e940b4 100644 --- a/thread-state_test.go +++ b/thread-state_test.go @@ -19,4 +19,3 @@ func TestYieldToEachOtherViaThreadStates(t *testing.T) { threadState.waitFor(stateActive) assert.True(t, threadState.is(stateActive)) } - diff --git a/worker-thread.go b/worker-thread.go index 4d2907606..4d83f1cc6 100644 --- a/worker-thread.go +++ b/worker-thread.go @@ -3,9 +3,9 @@ package frankenphp // #include "frankenphp.h" import "C" import ( + "fmt" "net/http" "path/filepath" - "fmt" "time" "go.uber.org/zap" @@ -16,30 +16,30 @@ import ( // executes the PHP worker script in a loop // implements the threadHandler interface type workerThread struct { - state *threadState - thread *phpThread - worker *worker - fakeRequest *http.Request + state *threadState + thread *phpThread + worker *worker + fakeRequest *http.Request workerRequest *http.Request - backoff *exponentialBackoff + backoff *exponentialBackoff } func convertToWorkerThread(thread *phpThread, worker *worker) { handler := &workerThread{ - state: thread.state, + state: thread.state, thread: thread, worker: worker, backoff: &exponentialBackoff{ - maxBackoff: 1 * time.Second, - minBackoff: 100 * time.Millisecond, - maxConsecutiveFailures: 6, - }, + maxBackoff: 1 * time.Second, + minBackoff: 100 * time.Millisecond, + maxConsecutiveFailures: 6, + }, } thread.handler = handler thread.requestChan = make(chan *http.Request) worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() thread.state.set(stateActive) } @@ -68,65 +68,15 @@ func (handler *workerThread) beforeScriptExecution() string { case stateRestarting: handler.state.set(stateYielding) handler.state.waitFor(stateReady, stateShuttingDown) - return handler.beforeScriptExecution() - case stateReady, stateActive: + return handler.beforeScriptExecution() + case stateReady, stateActive: setUpWorkerScript(handler, handler.worker) return handler.worker.fileName } - // TODO: panic? + // TODO: panic? return "" } -func (handler *workerThread) waitForWorkerRequest() bool { - - if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { - c.Write(zap.String("worker", handler.worker.fileName)) - } - - if handler.state.is(stateActive) { - metrics.ReadyWorker(handler.worker.fileName) - handler.state.set(stateReady) - } - - var r *http.Request - select { - case <-handler.thread.drainChan: - if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { - c.Write(zap.String("worker", handler.worker.fileName)) - } - - // execute opcache_reset if the restart was triggered by the watcher - if watcherIsEnabled && handler.state.is(stateRestarting) { - C.frankenphp_reset_opcache() - } - - return false - case r = <-handler.thread.requestChan: - case r = <-handler.worker.requestChan: - } - - handler.workerRequest = r - - if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { - c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI)) - } - - if err := updateServerContext(handler.thread, r, false, true); err != nil { - // Unexpected error - if c := logger.Check(zapcore.DebugLevel, "unexpected error"); c != nil { - c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) - } - fc := r.Context().Value(contextKey).(*FrankenPHPContext) - rejectRequest(fc.responseWriter, err.Error()) - maybeCloseContext(fc) - handler.workerRequest = nil - handler.thread.Unpin() - - return handler.waitForWorkerRequest() - } - return true -} - // return true if the worker should continue to run func (handler *workerThread) afterScriptExecution(exitStatus int) bool { tearDownWorkerScript(handler, exitStatus) @@ -134,15 +84,15 @@ func (handler *workerThread) afterScriptExecution(exitStatus int) bool { switch currentState { case stateDrain: handler.thread.requestChan = make(chan *http.Request) - return true + return true case stateShuttingDown: return false } return true } -func (handler *workerThread) onShutdown(){ - handler.state.set(stateDone) +func (handler *workerThread) onShutdown() { + handler.state.set(stateDone) } func setUpWorkerScript(handler *workerThread, worker *worker) { @@ -213,11 +163,61 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { // on exit status 1 we apply an exponential backoff when restarting metrics.StopWorker(worker.fileName, StopReasonCrash) if handler.backoff.recordFailure() { - if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) - } - logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", handler.backoff.failureCount)) - } + if !watcherIsEnabled { + panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) + } + logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", handler.backoff.failureCount)) + } +} + +func (handler *workerThread) waitForWorkerRequest() bool { + + if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { + c.Write(zap.String("worker", handler.worker.fileName)) + } + + if handler.state.is(stateActive) { + metrics.ReadyWorker(handler.worker.fileName) + handler.state.set(stateReady) + } + + var r *http.Request + select { + case <-handler.thread.drainChan: + if c := logger.Check(zapcore.DebugLevel, "shutting down"); c != nil { + c.Write(zap.String("worker", handler.worker.fileName)) + } + + // execute opcache_reset if the restart was triggered by the watcher + if watcherIsEnabled && handler.state.is(stateRestarting) { + C.frankenphp_reset_opcache() + } + + return false + case r = <-handler.thread.requestChan: + case r = <-handler.worker.requestChan: + } + + handler.workerRequest = r + + if c := logger.Check(zapcore.DebugLevel, "request handling started"); c != nil { + c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI)) + } + + if err := updateServerContext(handler.thread, r, false, true); err != nil { + // Unexpected error + if c := logger.Check(zapcore.DebugLevel, "unexpected error"); c != nil { + c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) + } + fc := r.Context().Value(contextKey).(*FrankenPHPContext) + rejectRequest(fc.responseWriter, err.Error()) + maybeCloseContext(fc) + handler.workerRequest = nil + handler.thread.Unpin() + + return handler.waitForWorkerRequest() + } + return true } //export go_frankenphp_worker_handle_request_start @@ -252,4 +252,4 @@ func go_frankenphp_finish_php_request(threadIndex C.uintptr_t) { if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { c.Write(zap.String("url", r.RequestURI)) } -} \ No newline at end of file +} diff --git a/worker.go b/worker.go index 8ee25cff0..b2646e9b0 100644 --- a/worker.go +++ b/worker.go @@ -22,7 +22,6 @@ type worker struct { threadMutex sync.RWMutex } - var ( workers map[string]*worker workersDone chan interface{} @@ -105,12 +104,12 @@ func restartWorkers() { stopWorkers() ready.Wait() for _, worker := range workers { - for _, thread := range worker.threads { - thread.drainChan = make(chan struct{}) - thread.state.set(stateReady) - } + for _, thread := range worker.threads { + thread.drainChan = make(chan struct{}) + thread.state.set(stateReady) + } worker.threadMutex.RUnlock() - } + } workersDone = make(chan interface{}) } @@ -150,4 +149,3 @@ func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { <-fc.done metrics.StopWorkerRequest(worker.fileName, time.Since(fc.startedAt)) } - From 02b73b169632bfc9ebdebed26e65670be25aae39 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 11:31:52 +0100 Subject: [PATCH 43/64] C formatting. --- frankenphp.c | 6 +++--- frankenphp_arginfo.h | 31 +++++++++++++------------------ 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index c4739e8e9..292156881 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -835,15 +835,15 @@ static void *php_thread(void *arg) { while (true) { char *scriptName = go_frankenphp_before_script_execution(thread_index); - int exit_status = 0; + int exit_status = 0; // if the script name is not empty, execute the PHP script if (strlen(scriptName) != 0) { exit_status = frankenphp_execute_script(scriptName); } // if go signals to stop, break the loop - if(!go_frankenphp_after_script_execution(thread_index, exit_status)){ - break; + if (!go_frankenphp_after_script_execution(thread_index, exit_status)) { + break; } } diff --git a/frankenphp_arginfo.h b/frankenphp_arginfo.h index ec97502e7..cecffd88d 100644 --- a/frankenphp_arginfo.h +++ b/frankenphp_arginfo.h @@ -36,22 +36,17 @@ ZEND_FUNCTION(frankenphp_finish_request); ZEND_FUNCTION(frankenphp_request_headers); ZEND_FUNCTION(frankenphp_response_headers); +// clang-format off static const zend_function_entry ext_functions[] = { - ZEND_FE(frankenphp_handle_request, arginfo_frankenphp_handle_request) - ZEND_FE(headers_send, arginfo_headers_send) ZEND_FE( - frankenphp_finish_request, arginfo_frankenphp_finish_request) - ZEND_FALIAS(fastcgi_finish_request, frankenphp_finish_request, - arginfo_fastcgi_finish_request) - ZEND_FE(frankenphp_request_headers, - arginfo_frankenphp_request_headers) - ZEND_FALIAS(apache_request_headers, - frankenphp_request_headers, - arginfo_apache_request_headers) - ZEND_FALIAS(getallheaders, frankenphp_request_headers, - arginfo_getallheaders) - ZEND_FE(frankenphp_response_headers, - arginfo_frankenphp_response_headers) - ZEND_FALIAS(apache_response_headers, - frankenphp_response_headers, - arginfo_apache_response_headers) - ZEND_FE_END}; + ZEND_FE(frankenphp_handle_request, arginfo_frankenphp_handle_request) + ZEND_FE(headers_send, arginfo_headers_send) + ZEND_FE(frankenphp_finish_request, arginfo_frankenphp_finish_request) + ZEND_FALIAS(fastcgi_finish_request, frankenphp_finish_request, arginfo_fastcgi_finish_request) + ZEND_FE(frankenphp_request_headers, arginfo_frankenphp_request_headers) + ZEND_FALIAS(apache_request_headers, frankenphp_request_headers, arginfo_apache_request_headers) + ZEND_FALIAS(getallheaders, frankenphp_request_headers, arginfo_getallheaders) + ZEND_FE(frankenphp_response_headers, arginfo_frankenphp_response_headers) + ZEND_FALIAS(apache_response_headers, frankenphp_response_headers, arginfo_apache_response_headers) + ZEND_FE_END +}; +// clang-format on \ No newline at end of file From 421904e8794a2bd3a97a307595e506aa6b77691c Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 11:45:15 +0100 Subject: [PATCH 44/64] More cleanup. --- frankenphp.go | 4 ---- main-thread.go | 2 ++ worker.go | 11 +---------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/frankenphp.go b/frankenphp.go index 43ae16a7b..809e4af7d 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -477,10 +477,6 @@ func ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) error return nil } -func handleRequest(thread *phpThread) { - -} - func maybeCloseContext(fc *FrankenPHPContext) { fc.closed.Do(func() { close(fc.done) diff --git a/main-thread.go b/main-thread.go index 687455945..bf117d193 100644 --- a/main-thread.go +++ b/main-thread.go @@ -7,6 +7,8 @@ import ( "sync" ) +// represents the main PHP thread +// the thread needs to keep running as long as all other threads are running type mainPHPThread struct { state *threadState done chan struct{} diff --git a/worker.go b/worker.go index b2646e9b0..ba7ec9fb6 100644 --- a/worker.go +++ b/worker.go @@ -24,13 +24,11 @@ type worker struct { var ( workers map[string]*worker - workersDone chan interface{} watcherIsEnabled bool ) func initWorkers(opt []workerOpt) error { workers = make(map[string]*worker, len(opt)) - workersDone = make(chan interface{}) directoriesToWatch := getDirectoriesToWatch(opt) watcherIsEnabled = len(directoriesToWatch) > 0 @@ -45,7 +43,7 @@ func initWorkers(opt []workerOpt) error { } } - if len(directoriesToWatch) == 0 { + if !watcherIsEnabled { return nil } @@ -78,13 +76,8 @@ func newWorker(o workerOpt) (*worker, error) { return w, nil } -func stopWorkers() { - close(workersDone) -} - func drainWorkers() { watcher.DrainWatcher() - stopWorkers() } func restartWorkers() { @@ -101,7 +94,6 @@ func restartWorkers() { }(thread) } } - stopWorkers() ready.Wait() for _, worker := range workers { for _, thread := range worker.threads { @@ -110,7 +102,6 @@ func restartWorkers() { } worker.threadMutex.RUnlock() } - workersDone = make(chan interface{}) } func getDirectoriesToWatch(workerOpts []workerOpt) []string { From cca2a00ac6755cf2b9ea88900c1ac1825675c456 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 14:59:34 +0100 Subject: [PATCH 45/64] Allows for clean state transitions. --- frankenphp.c | 15 ++--- inactive-thread.go | 46 ---------------- main-thread_test.go | 20 ------- main-thread.go => phpmainthread.go | 28 +++++----- phpmainthread_test.go | 76 ++++++++++++++++++++++++++ phpthread.go | 24 +++++--- thread-inactive.go | 42 ++++++++++++++ regular-thread.go => thread-regular.go | 66 ++++++++-------------- thread-state.go | 42 ++++++++++---- thread-state_test.go | 6 +- worker-thread.go => thread-worker.go | 65 +++++++--------------- worker.go | 17 ++++++ 12 files changed, 252 insertions(+), 195 deletions(-) delete mode 100644 inactive-thread.go delete mode 100644 main-thread_test.go rename main-thread.go => phpmainthread.go (86%) create mode 100644 phpmainthread_test.go create mode 100644 thread-inactive.go rename regular-thread.go => thread-regular.go (62%) rename worker-thread.go => thread-worker.go (85%) diff --git a/frankenphp.c b/frankenphp.c index 292156881..b4cde79cd 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -835,16 +835,13 @@ static void *php_thread(void *arg) { while (true) { char *scriptName = go_frankenphp_before_script_execution(thread_index); - int exit_status = 0; - // if the script name is not empty, execute the PHP script - if (strlen(scriptName) != 0) { - exit_status = frankenphp_execute_script(scriptName); - } - // if go signals to stop, break the loop - if (!go_frankenphp_after_script_execution(thread_index, exit_status)) { - break; - } + if (scriptName == NULL) { + break; + } + + int exit_status = frankenphp_execute_script(scriptName); + go_frankenphp_after_script_execution(thread_index, exit_status); } go_frankenphp_release_known_variable_keys(thread_index); diff --git a/inactive-thread.go b/inactive-thread.go deleted file mode 100644 index 80921ebcb..000000000 --- a/inactive-thread.go +++ /dev/null @@ -1,46 +0,0 @@ -package frankenphp - -import ( - "net/http" - "strconv" -) - -// representation of a thread with no work assigned to it -// implements the threadHandler interface -type inactiveThread struct { - state *threadState -} - -func convertToInactiveThread(thread *phpThread) { - thread.handler = &inactiveThread{state: thread.state} -} - -func (t *inactiveThread) isReadyToTransition() bool { - return true -} - -func (thread *inactiveThread) getActiveRequest() *http.Request { - panic("idle threads have no requests") -} - -func (thread *inactiveThread) beforeScriptExecution() string { - // no script execution for inactive threads - return "" -} - -func (thread *inactiveThread) afterScriptExecution(exitStatus int) bool { - thread.state.set(stateInactive) - // wait for external signal to start or shut down - thread.state.waitFor(stateActive, stateShuttingDown) - switch thread.state.get() { - case stateActive: - return true - case stateShuttingDown: - return false - } - panic("unexpected state: " + strconv.Itoa(int(thread.state.get()))) -} - -func (thread *inactiveThread) onShutdown() { - thread.state.set(stateDone) -} diff --git a/main-thread_test.go b/main-thread_test.go deleted file mode 100644 index 74aa75145..000000000 --- a/main-thread_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package frankenphp - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "go.uber.org/zap" -) - -func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { - logger = zap.NewNop() // the logger needs to not be nil - assert.NoError(t, initPHPThreads(1)) // reserve 1 thread - - assert.Len(t, phpThreads, 1) - assert.Equal(t, 0, phpThreads[0].threadIndex) - assert.True(t, phpThreads[0].state.is(stateInactive)) - - drainPHPThreads() - assert.Nil(t, phpThreads) -} diff --git a/main-thread.go b/phpmainthread.go similarity index 86% rename from main-thread.go rename to phpmainthread.go index bf117d193..e9378070a 100644 --- a/main-thread.go +++ b/phpmainthread.go @@ -4,12 +4,13 @@ package frankenphp import "C" import ( "fmt" + "net/http" "sync" ) // represents the main PHP thread // the thread needs to keep running as long as all other threads are running -type mainPHPThread struct { +type phpMainThread struct { state *threadState done chan struct{} numThreads int @@ -17,34 +18,36 @@ type mainPHPThread struct { var ( phpThreads []*phpThread - mainThread *mainPHPThread + mainThread *phpMainThread ) // reserve a fixed number of PHP threads on the go side func initPHPThreads(numThreads int) error { - mainThread = &mainPHPThread{ + mainThread = &phpMainThread{ state: newThreadState(), done: make(chan struct{}), numThreads: numThreads, } phpThreads = make([]*phpThread, numThreads) + if err := mainThread.start(); err != nil { + return err + } + + // initialize all threads as inactive for i := 0; i < numThreads; i++ { phpThreads[i] = &phpThread{ threadIndex: i, drainChan: make(chan struct{}), + requestChan: make(chan *http.Request), state: newThreadState(), } convertToInactiveThread(phpThreads[i]) } - if err := mainThread.start(); err != nil { - return err - } - // initialize all threads as inactive + // start the underlying C threads ready := sync.WaitGroup{} ready.Add(numThreads) - for _, thread := range phpThreads { go func() { if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { @@ -54,7 +57,6 @@ func initPHPThreads(numThreads int) error { ready.Done() }() } - ready.Wait() return nil @@ -80,17 +82,17 @@ func drainPHPThreads() { phpThreads = nil } -func (mainThread *mainPHPThread) start() error { +func (mainThread *phpMainThread) start() error { if C.frankenphp_new_main_thread(C.int(mainThread.numThreads)) != 0 { return MainThreadCreationError } - mainThread.state.waitFor(stateActive) + mainThread.state.waitFor(stateReady) return nil } func getInactivePHPThread() *phpThread { for _, thread := range phpThreads { - if thread.handler.isReadyToTransition() { + if thread.state.is(stateInactive) { return thread } } @@ -99,7 +101,7 @@ func getInactivePHPThread() *phpThread { //export go_frankenphp_main_thread_is_ready func go_frankenphp_main_thread_is_ready() { - mainThread.state.set(stateActive) + mainThread.state.set(stateReady) mainThread.state.waitFor(stateShuttingDown) } diff --git a/phpmainthread_test.go b/phpmainthread_test.go new file mode 100644 index 000000000..f9f46cc15 --- /dev/null +++ b/phpmainthread_test.go @@ -0,0 +1,76 @@ +package frankenphp + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { + logger = zap.NewNop() // the logger needs to not be nil + assert.NoError(t, initPHPThreads(1)) // reserve 1 thread + + assert.Len(t, phpThreads, 1) + assert.Equal(t, 0, phpThreads[0].threadIndex) + assert.True(t, phpThreads[0].state.is(stateInactive)) + + drainPHPThreads() + assert.Nil(t, phpThreads) +} + +func TestTransition2RegularThreadsToWorkerThreadsAndBack(t *testing.T) { + numThreads := 2 + logger, _ = zap.NewDevelopment() + assert.NoError(t, initPHPThreads(numThreads)) + + // transition to worker thread + for i := 0; i < numThreads; i++ { + convertToRegularThread(phpThreads[i]) + assert.IsType(t, ®ularThread{}, phpThreads[i].handler) + } + + // transition to worker thread + worker := getDummyWorker() + for i := 0; i < numThreads; i++ { + convertToWorkerThread(phpThreads[i], worker) + assert.IsType(t, &workerThread{}, phpThreads[i].handler) + } + assert.Len(t, worker.threads, numThreads) + + // transition back to regular thread + for i := 0; i < numThreads; i++ { + convertToRegularThread(phpThreads[i]) + assert.IsType(t, ®ularThread{}, phpThreads[i].handler) + } + assert.Len(t, worker.threads, 0) + + drainPHPThreads() + assert.Nil(t, phpThreads) +} + +func TestTransitionAThreadBetween2DifferentWorkers(t *testing.T) { + logger, _ = zap.NewDevelopment() + assert.NoError(t, initPHPThreads(1)) + + // convert to first worker thread + firstWorker := getDummyWorker() + convertToWorkerThread(phpThreads[0], firstWorker) + firstHandler := phpThreads[0].handler.(*workerThread) + assert.Same(t, firstWorker, firstHandler.worker) + + // convert to second worker thread + secondWorker := getDummyWorker() + convertToWorkerThread(phpThreads[0], secondWorker) + secondHandler := phpThreads[0].handler.(*workerThread) + assert.Same(t, secondWorker, secondHandler.worker) + + drainPHPThreads() + assert.Nil(t, phpThreads) +} + +func getDummyWorker() *worker { + path, _ := filepath.Abs("./testdata/index.php") + return &worker{fileName: path} +} diff --git a/phpthread.go b/phpthread.go index 465a1eb17..5ee1ff34a 100644 --- a/phpthread.go +++ b/phpthread.go @@ -6,6 +6,8 @@ import ( "net/http" "runtime" "unsafe" + + "go.uber.org/zap" ) // representation of the actual underlying PHP thread @@ -24,19 +26,23 @@ type phpThread struct { // interface that defines how the callbacks from the C thread should be handled type threadHandler interface { beforeScriptExecution() string - afterScriptExecution(exitStatus int) bool - onShutdown() + afterScriptExecution(exitStatus int) getActiveRequest() *http.Request - isReadyToTransition() bool } func (thread *phpThread) getActiveRequest() *http.Request { return thread.handler.getActiveRequest() } +// change the thread handler safely func (thread *phpThread) setHandler(handler threadHandler) { + logger.Debug("transitioning thread", zap.Int("threadIndex", thread.threadIndex)) + thread.state.set(stateTransitionRequested) + close(thread.drainChan) + thread.state.waitFor(stateTransitionInProgress) thread.handler = handler - thread.state.set(stateActive) + thread.drainChan = make(chan struct{}) + thread.state.set(stateTransitionComplete) } // Pin a string that is not null-terminated @@ -56,19 +62,23 @@ func (thread *phpThread) pinCString(s string) *C.char { func go_frankenphp_before_script_execution(threadIndex C.uintptr_t) *C.char { thread := phpThreads[threadIndex] scriptName := thread.handler.beforeScriptExecution() + + // if no scriptName is passed, shut down + if scriptName == "" { + return nil + } // return the name of the PHP script that should be executed return thread.pinCString(scriptName) } //export go_frankenphp_after_script_execution -func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C.int) C.bool { +func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C.int) { thread := phpThreads[threadIndex] if exitStatus < 0 { panic(ScriptExecutionError) } - shouldContinueExecution := thread.handler.afterScriptExecution(int(exitStatus)) + thread.handler.afterScriptExecution(int(exitStatus)) thread.Unpin() - return C.bool(shouldContinueExecution) } //export go_frankenphp_on_thread_shutdown diff --git a/thread-inactive.go b/thread-inactive.go new file mode 100644 index 000000000..311ecabed --- /dev/null +++ b/thread-inactive.go @@ -0,0 +1,42 @@ +package frankenphp + +import ( + "net/http" +) + +// representation of a thread with no work assigned to it +// implements the threadHandler interface +type inactiveThread struct { + thread *phpThread +} + +func convertToInactiveThread(thread *phpThread) { + thread.handler = &inactiveThread{thread: thread} +} + +func (thread *inactiveThread) getActiveRequest() *http.Request { + panic("inactive threads have no requests") +} + +func (handler *inactiveThread) beforeScriptExecution() string { + thread := handler.thread + thread.state.set(stateInactive) + + // wait for external signal to start or shut down + thread.state.waitFor(stateTransitionRequested, stateShuttingDown) + switch thread.state.get() { + case stateTransitionRequested: + thread.state.set(stateTransitionInProgress) + thread.state.waitFor(stateTransitionComplete, stateShuttingDown) + // execute beforeScriptExecution of the new handler + return thread.handler.beforeScriptExecution() + case stateShuttingDown: + // signal to stop + return "" + } + panic("unexpected state: " + thread.state.name()) +} + +func (thread *inactiveThread) afterScriptExecution(exitStatus int) { + panic("inactive threads should not execute scripts") +} diff --git a/regular-thread.go b/thread-regular.go similarity index 62% rename from regular-thread.go rename to thread-regular.go index ef82c568d..ee9839d2c 100644 --- a/regular-thread.go +++ b/thread-regular.go @@ -16,59 +16,47 @@ type regularThread struct { } func convertToRegularThread(thread *phpThread) { - thread.handler = ®ularThread{ + thread.setHandler(®ularThread{ thread: thread, state: thread.state, - } - thread.state.set(stateActive) -} - -func (t *regularThread) isReadyToTransition() bool { - return false -} - -func (handler *regularThread) getActiveRequest() *http.Request { - return handler.activeRequest + }) } // return the name of the script or an empty string if no script should be executed func (handler *regularThread) beforeScriptExecution() string { - currentState := handler.state.get() - switch currentState { - case stateInactive: - handler.state.waitFor(stateActive, stateShuttingDown) - return handler.beforeScriptExecution() + switch handler.state.get() { + case stateTransitionRequested: + thread := handler.thread + thread.state.set(stateTransitionInProgress) + thread.state.waitFor(stateTransitionComplete, stateShuttingDown) + // execute beforeScriptExecution of the new handler + return thread.handler.beforeScriptExecution() + case stateTransitionComplete: + handler.state.set(stateReady) + return handler.waitForRequest() case stateShuttingDown: + // signal to stop return "" - case stateReady, stateActive: - return handler.waitForScriptExecution() + case stateReady: + return handler.waitForRequest() } - return "" + panic("unexpected state: " + handler.state.name()) } // return true if the worker should continue to run -func (handler *regularThread) afterScriptExecution(exitStatus int) bool { +func (handler *regularThread) afterScriptExecution(exitStatus int) { handler.afterRequest(exitStatus) - - currentState := handler.state.get() - switch currentState { - case stateDrain: - return true - case stateShuttingDown: - return false - } - return true } -func (handler *regularThread) onShutdown() { - handler.state.set(stateDone) +func (handler *regularThread) getActiveRequest() *http.Request { + return handler.activeRequest } -func (handler *regularThread) waitForScriptExecution() string { +func (handler *regularThread) waitForRequest() string { select { case <-handler.thread.drainChan: - // no script should be executed if the server is shutting down - return "" + // go back to beforeScriptExecution + return handler.beforeScriptExecution() case r := <-requestChan: handler.activeRequest = r @@ -78,8 +66,8 @@ func (handler *regularThread) waitForScriptExecution() string { rejectRequest(fc.responseWriter, err.Error()) handler.afterRequest(0) handler.thread.Unpin() - // no script should be executed if the request was rejected - return "" + // go back to beforeScriptExecution + return handler.beforeScriptExecution() } // set the scriptName that should be executed @@ -88,12 +76,6 @@ func (handler *regularThread) waitForScriptExecution() string { } func (handler *regularThread) afterRequest(exitStatus int) { - - // if the request is nil, no script was executed - if handler.activeRequest == nil { - return - } - fc := handler.activeRequest.Context().Value(contextKey).(*FrankenPHPContext) fc.exitStatus = exitStatus maybeCloseContext(fc) diff --git a/thread-state.go b/thread-state.go index 5ca9443dd..28a9085ea 100644 --- a/thread-state.go +++ b/thread-state.go @@ -2,22 +2,28 @@ package frankenphp import ( "slices" + "strconv" "sync" ) type stateID int const ( + // initial state stateBooting stateID = iota stateInactive - stateActive stateReady - stateBusy stateShuttingDown stateDone + + // states necessary for restarting workers stateRestarting - stateDrain stateYielding + + // states necessary for transitioning + stateTransitionRequested + stateTransitionInProgress + stateTransitionComplete ) type threadState struct { @@ -39,16 +45,26 @@ func newThreadState() *threadState { } } -func (h *threadState) is(state stateID) bool { - h.mu.RLock() - defer h.mu.RUnlock() - return h.currentState == state +func (ts *threadState) is(state stateID) bool { + ts.mu.RLock() + defer ts.mu.RUnlock() + return ts.currentState == state } -func (h *threadState) get() stateID { - h.mu.RLock() - defer h.mu.RUnlock() - return h.currentState +func (ts *threadState) compareAndSwap(compareTo stateID, swapTo stateID) bool { + ts.mu.Lock() + defer ts.mu.Unlock() + if ts.currentState == compareTo { + ts.currentState = swapTo + return true + } + return false +} + +func (ts *threadState) get() stateID { + ts.mu.RLock() + defer ts.mu.RUnlock() + return ts.currentState } func (h *threadState) set(nextState stateID) { @@ -72,6 +88,10 @@ func (h *threadState) set(nextState stateID) { h.subscribers = newSubscribers } +func (ts *threadState) name() string { + return "state:" + strconv.Itoa(int(ts.get())) +} + // block until the thread reaches a certain state func (h *threadState) waitFor(states ...stateID) { h.mu.Lock() diff --git a/thread-state_test.go b/thread-state_test.go index f71e940b4..28bb3a693 100644 --- a/thread-state_test.go +++ b/thread-state_test.go @@ -12,10 +12,10 @@ func TestYieldToEachOtherViaThreadStates(t *testing.T) { go func() { threadState.waitFor(stateInactive) assert.True(t, threadState.is(stateInactive)) - threadState.set(stateActive) + threadState.set(stateReady) }() threadState.set(stateInactive) - threadState.waitFor(stateActive) - assert.True(t, threadState.is(stateActive)) + threadState.waitFor(stateReady) + assert.True(t, threadState.is(stateReady)) } diff --git a/worker-thread.go b/thread-worker.go similarity index 85% rename from worker-thread.go rename to thread-worker.go index 4d83f1cc6..be70334d7 100644 --- a/worker-thread.go +++ b/thread-worker.go @@ -25,7 +25,7 @@ type workerThread struct { } func convertToWorkerThread(thread *phpThread, worker *worker) { - handler := &workerThread{ + thread.setHandler(&workerThread{ state: thread.state, thread: thread, worker: worker, @@ -34,14 +34,11 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { minBackoff: 100 * time.Millisecond, maxConsecutiveFailures: 6, }, + }) + worker.addThread(thread) + if worker.fileName == "" { + panic("worker script is empty") } - thread.handler = handler - thread.requestChan = make(chan *http.Request) - worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() - - thread.state.set(stateActive) } func (handler *workerThread) getActiveRequest() *http.Request { @@ -52,47 +49,33 @@ func (handler *workerThread) getActiveRequest() *http.Request { return handler.fakeRequest } -func (t *workerThread) isReadyToTransition() bool { - return false -} - // return the name of the script or an empty string if no script should be executed func (handler *workerThread) beforeScriptExecution() string { - currentState := handler.state.get() - switch currentState { - case stateInactive: - handler.state.waitFor(stateActive, stateShuttingDown) - return handler.beforeScriptExecution() + switch handler.state.get() { + case stateTransitionRequested: + thread := handler.thread + handler.worker.removeThread(handler.thread) + thread.state.set(stateTransitionInProgress) + thread.state.waitFor(stateTransitionComplete, stateShuttingDown) + + // execute beforeScriptExecution of the new handler + return thread.handler.beforeScriptExecution() case stateShuttingDown: + // signal to stop return "" case stateRestarting: handler.state.set(stateYielding) handler.state.waitFor(stateReady, stateShuttingDown) return handler.beforeScriptExecution() - case stateReady, stateActive: + case stateReady, stateTransitionComplete: setUpWorkerScript(handler, handler.worker) return handler.worker.fileName } - // TODO: panic? - return "" + panic("unexpected state: " + handler.state.name()) } -// return true if the worker should continue to run -func (handler *workerThread) afterScriptExecution(exitStatus int) bool { +func (handler *workerThread) afterScriptExecution(exitStatus int) { tearDownWorkerScript(handler, exitStatus) - currentState := handler.state.get() - switch currentState { - case stateDrain: - handler.thread.requestChan = make(chan *http.Request) - return true - case stateShuttingDown: - return false - } - return true -} - -func (handler *workerThread) onShutdown() { - handler.state.set(stateDone) } func setUpWorkerScript(handler *workerThread, worker *worker) { @@ -126,11 +109,7 @@ func setUpWorkerScript(handler *workerThread, worker *worker) { func tearDownWorkerScript(handler *workerThread, exitStatus int) { - // if the fake request is nil, no script was executed - if handler.fakeRequest == nil { - return - } - + logger.Info("tear down worker script") // if the worker request is not nil, the script might have crashed // make sure to close the worker request context if handler.workerRequest != nil { @@ -171,14 +150,12 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { } func (handler *workerThread) waitForWorkerRequest() bool { - if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { c.Write(zap.String("worker", handler.worker.fileName)) } - if handler.state.is(stateActive) { + if handler.state.compareAndSwap(stateTransitionComplete, stateReady) { metrics.ReadyWorker(handler.worker.fileName) - handler.state.set(stateReady) } var r *http.Request @@ -205,7 +182,7 @@ func (handler *workerThread) waitForWorkerRequest() bool { } if err := updateServerContext(handler.thread, r, false, true); err != nil { - // Unexpected error + // Unexpected error or invalid request if c := logger.Check(zapcore.DebugLevel, "unexpected error"); c != nil { c.Write(zap.String("worker", handler.worker.fileName), zap.String("url", r.RequestURI), zap.Error(err)) } diff --git a/worker.go b/worker.go index ba7ec9fb6..0ef61d4c0 100644 --- a/worker.go +++ b/worker.go @@ -140,3 +140,20 @@ func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { <-fc.done metrics.StopWorkerRequest(worker.fileName, time.Since(fc.startedAt)) } + +func (worker *worker) addThread(thread *phpThread) { + worker.threadMutex.Lock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() +} + +func (worker *worker) removeThread(thread *phpThread) { + worker.threadMutex.Lock() + for i, t := range worker.threads { + if t == thread { + worker.threads = append(worker.threads[:i], worker.threads[i+1:]...) + break + } + } + worker.threadMutex.Unlock() +} From ec8aeb7bd11fddc62fe4a997fbb244fc06c3c3d2 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 15:14:32 +0100 Subject: [PATCH 46/64] Adds state tests. --- thread-state.go => state.go | 0 state_test.go | 48 +++++++++++++++++++++++++++++++++++++ thread-state_test.go | 21 ---------------- thread-worker.go | 4 ++-- worker.go | 38 +++++++++++++---------------- 5 files changed, 67 insertions(+), 44 deletions(-) rename thread-state.go => state.go (100%) create mode 100644 state_test.go delete mode 100644 thread-state_test.go diff --git a/thread-state.go b/state.go similarity index 100% rename from thread-state.go rename to state.go diff --git a/state_test.go b/state_test.go new file mode 100644 index 000000000..47b68d410 --- /dev/null +++ b/state_test.go @@ -0,0 +1,48 @@ +package frankenphp + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func Test2GoroutinesYieldToEachOtherViaStates(t *testing.T) { + threadState := &threadState{currentState: stateBooting} + + go func() { + threadState.waitFor(stateInactive) + assert.True(t, threadState.is(stateInactive)) + threadState.set(stateReady) + }() + + threadState.set(stateInactive) + threadState.waitFor(stateReady) + assert.True(t, threadState.is(stateReady)) +} + +func TestStateShouldHaveCorrectAmountOfSubscribers(t *testing.T) { + threadState := &threadState{currentState: stateBooting} + + // 3 subscribers waiting for different states + go threadState.waitFor(stateInactive) + go threadState.waitFor(stateInactive, stateShuttingDown) + go threadState.waitFor(stateShuttingDown) + + time.Sleep(1 * time.Millisecond) + assertNumberOfSubscribers(t, threadState, 3) + + threadState.set(stateInactive) + time.Sleep(1 * time.Millisecond) + assertNumberOfSubscribers(t, threadState, 1) + + threadState.set(stateShuttingDown) + time.Sleep(1 * time.Millisecond) + assertNumberOfSubscribers(t, threadState, 0) +} + +func assertNumberOfSubscribers(t *testing.T, threadState *threadState, expected int) { + threadState.mu.RLock() + assert.Len(t, threadState.subscribers, expected) + threadState.mu.RUnlock() +} diff --git a/thread-state_test.go b/thread-state_test.go deleted file mode 100644 index 28bb3a693..000000000 --- a/thread-state_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package frankenphp - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestYieldToEachOtherViaThreadStates(t *testing.T) { - threadState := &threadState{currentState: stateBooting} - - go func() { - threadState.waitFor(stateInactive) - assert.True(t, threadState.is(stateInactive)) - threadState.set(stateReady) - }() - - threadState.set(stateInactive) - threadState.waitFor(stateReady) - assert.True(t, threadState.is(stateReady)) -} diff --git a/thread-worker.go b/thread-worker.go index be70334d7..75d2433f1 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -35,7 +35,7 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { maxConsecutiveFailures: 6, }, }) - worker.addThread(thread) + worker.attachThread(thread) if worker.fileName == "" { panic("worker script is empty") } @@ -54,7 +54,7 @@ func (handler *workerThread) beforeScriptExecution() string { switch handler.state.get() { case stateTransitionRequested: thread := handler.thread - handler.worker.removeThread(handler.thread) + handler.worker.detachThread(handler.thread) thread.state.set(stateTransitionInProgress) thread.state.waitFor(stateTransitionComplete, stateShuttingDown) diff --git a/worker.go b/worker.go index 0ef61d4c0..374361591 100644 --- a/worker.go +++ b/worker.go @@ -39,7 +39,8 @@ func initWorkers(opt []workerOpt) error { return err } for i := 0; i < worker.num; i++ { - worker.startNewThread() + thread := getInactivePHPThread() + convertToWorkerThread(thread, worker) } } @@ -112,9 +113,21 @@ func getDirectoriesToWatch(workerOpts []workerOpt) []string { return directoriesToWatch } -func (worker *worker) startNewThread() { - thread := getInactivePHPThread() - convertToWorkerThread(thread, worker) +func (worker *worker) attachThread(thread *phpThread) { + worker.threadMutex.Lock() + worker.threads = append(worker.threads, thread) + worker.threadMutex.Unlock() +} + +func (worker *worker) detachThread(thread *phpThread) { + worker.threadMutex.Lock() + for i, t := range worker.threads { + if t == thread { + worker.threads = append(worker.threads[:i], worker.threads[i+1:]...) + break + } + } + worker.threadMutex.Unlock() } func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { @@ -140,20 +153,3 @@ func (worker *worker) handleRequest(r *http.Request, fc *FrankenPHPContext) { <-fc.done metrics.StopWorkerRequest(worker.fileName, time.Since(fc.startedAt)) } - -func (worker *worker) addThread(thread *phpThread) { - worker.threadMutex.Lock() - worker.threads = append(worker.threads, thread) - worker.threadMutex.Unlock() -} - -func (worker *worker) removeThread(thread *phpThread) { - worker.threadMutex.Lock() - for i, t := range worker.threads { - if t == thread { - worker.threads = append(worker.threads[:i], worker.threads[i+1:]...) - break - } - } - worker.threadMutex.Unlock() -} From b598bd344ffc0561487266c74bb6d1293cfbe826 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 17:59:50 +0100 Subject: [PATCH 47/64] Adds support for thread transitioning. --- phpmainthread.go | 10 +-- phpmainthread_test.go | 131 +++++++++++++++++++++++++------ phpthread.go | 22 +++++- testdata/sleep.php | 4 - testdata/transition-regular.php | 3 + testdata/transition-worker-1.php | 7 ++ testdata/transition-worker-2.php | 8 ++ 7 files changed, 145 insertions(+), 40 deletions(-) delete mode 100644 testdata/sleep.php create mode 100644 testdata/transition-regular.php create mode 100644 testdata/transition-worker-1.php create mode 100644 testdata/transition-worker-2.php diff --git a/phpmainthread.go b/phpmainthread.go index e9378070a..c0ffb1614 100644 --- a/phpmainthread.go +++ b/phpmainthread.go @@ -4,7 +4,6 @@ package frankenphp import "C" import ( "fmt" - "net/http" "sync" ) @@ -36,12 +35,7 @@ func initPHPThreads(numThreads int) error { // initialize all threads as inactive for i := 0; i < numThreads; i++ { - phpThreads[i] = &phpThread{ - threadIndex: i, - drainChan: make(chan struct{}), - requestChan: make(chan *http.Request), - state: newThreadState(), - } + phpThreads[i] = newPHPThread(i) convertToInactiveThread(phpThreads[i]) } @@ -66,6 +60,7 @@ func drainPHPThreads() { doneWG := sync.WaitGroup{} doneWG.Add(len(phpThreads)) for _, thread := range phpThreads { + thread.mu.Lock() thread.state.set(stateShuttingDown) close(thread.drainChan) } @@ -73,6 +68,7 @@ func drainPHPThreads() { for _, thread := range phpThreads { go func(thread *phpThread) { thread.state.waitFor(stateDone) + thread.mu.Unlock() doneWG.Done() }(thread) } diff --git a/phpmainthread_test.go b/phpmainthread_test.go index f9f46cc15..25e448f2c 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -1,8 +1,14 @@ package frankenphp import ( + "io" + "math/rand/v2" + "net/http/httptest" "path/filepath" + "sync" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "go.uber.org/zap" @@ -20,30 +26,23 @@ func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { assert.Nil(t, phpThreads) } -func TestTransition2RegularThreadsToWorkerThreadsAndBack(t *testing.T) { - numThreads := 2 - logger, _ = zap.NewDevelopment() - assert.NoError(t, initPHPThreads(numThreads)) +func TestTransitionRegularThreadToWorkerThread(t *testing.T) { + logger = zap.NewNop() + assert.NoError(t, initPHPThreads(1)) - // transition to worker thread - for i := 0; i < numThreads; i++ { - convertToRegularThread(phpThreads[i]) - assert.IsType(t, ®ularThread{}, phpThreads[i].handler) - } + // transition to regular thread + convertToRegularThread(phpThreads[0]) + assert.IsType(t, ®ularThread{}, phpThreads[0].handler) // transition to worker thread - worker := getDummyWorker() - for i := 0; i < numThreads; i++ { - convertToWorkerThread(phpThreads[i], worker) - assert.IsType(t, &workerThread{}, phpThreads[i].handler) - } - assert.Len(t, worker.threads, numThreads) + worker := getDummyWorker("worker-transition-1.php") + convertToWorkerThread(phpThreads[0], worker) + assert.IsType(t, &workerThread{}, phpThreads[0].handler) + assert.Len(t, worker.threads, 1) // transition back to regular thread - for i := 0; i < numThreads; i++ { - convertToRegularThread(phpThreads[i]) - assert.IsType(t, ®ularThread{}, phpThreads[i].handler) - } + convertToRegularThread(phpThreads[0]) + assert.IsType(t, ®ularThread{}, phpThreads[0].handler) assert.Len(t, worker.threads, 0) drainPHPThreads() @@ -51,26 +50,108 @@ func TestTransition2RegularThreadsToWorkerThreadsAndBack(t *testing.T) { } func TestTransitionAThreadBetween2DifferentWorkers(t *testing.T) { - logger, _ = zap.NewDevelopment() + logger = zap.NewNop() assert.NoError(t, initPHPThreads(1)) + firstWorker := getDummyWorker("worker-transition-1.php") + secondWorker := getDummyWorker("worker-transition-2.php") // convert to first worker thread - firstWorker := getDummyWorker() convertToWorkerThread(phpThreads[0], firstWorker) firstHandler := phpThreads[0].handler.(*workerThread) assert.Same(t, firstWorker, firstHandler.worker) + assert.Len(t, firstWorker.threads, 1) + assert.Len(t, secondWorker.threads, 0) // convert to second worker thread - secondWorker := getDummyWorker() convertToWorkerThread(phpThreads[0], secondWorker) secondHandler := phpThreads[0].handler.(*workerThread) assert.Same(t, secondWorker, secondHandler.worker) + assert.Len(t, firstWorker.threads, 0) + assert.Len(t, secondWorker.threads, 1) drainPHPThreads() assert.Nil(t, phpThreads) } -func getDummyWorker() *worker { - path, _ := filepath.Abs("./testdata/index.php") - return &worker{fileName: path} +func TestTransitionThreadsWhileDoingRequests(t *testing.T) { + numThreads := 10 + numRequestsPerThread := 100 + isRunning := atomic.Bool{} + isRunning.Store(true) + wg := sync.WaitGroup{} + worker1Path, _ := filepath.Abs("./testdata/transition-worker-1.php") + worker2Path, _ := filepath.Abs("./testdata/transition-worker-2.php") + + Init( + WithNumThreads(numThreads), + WithWorkers(worker1Path, 4, map[string]string{"ENV1": "foo"}, []string{}), + WithWorkers(worker2Path, 4, map[string]string{"ENV1": "foo"}, []string{}), + WithLogger(zap.NewNop()), + ) + + // randomly transition threads between regular and 2 worker threads + go func() { + for { + for i := 0; i < numThreads; i++ { + switch rand.IntN(3) { + case 0: + convertToRegularThread(phpThreads[i]) + case 1: + convertToWorkerThread(phpThreads[i], workers[worker1Path]) + case 2: + convertToWorkerThread(phpThreads[i], workers[worker2Path]) + } + time.Sleep(time.Millisecond) + if !isRunning.Load() { + return + } + } + } + }() + + // randomly do requests to the 3 endpoints + wg.Add(numThreads) + for i := 0; i < numThreads; i++ { + go func(i int) { + for j := 0; j < numRequestsPerThread; j++ { + switch rand.IntN(3) { + case 0: + assertRequestBody(t, "http://localhost/transition-worker-1.php", "Hello from worker 1") + case 1: + assertRequestBody(t, "http://localhost/transition-worker-2.php", "Hello from worker 2") + case 2: + assertRequestBody(t, "http://localhost/transition-regular.php", "Hello from regular thread") + } + } + wg.Done() + }(i) + } + + wg.Wait() + isRunning.Store(false) + Shutdown() +} + +func getDummyWorker(fileName string) *worker { + if workers == nil { + workers = make(map[string]*worker) + } + absFileName, _ := filepath.Abs("./testdata/" + fileName) + worker, _ := newWorker(workerOpt{ + fileName: absFileName, + num: 1, + }) + return worker +} + +func assertRequestBody(t *testing.T, url string, expected string) { + r := httptest.NewRequest("GET", url, nil) + w := httptest.NewRecorder() + req, err := NewRequestWithContext(r, WithRequestDocumentRoot("/go/src/app/testdata", false)) + assert.NoError(t, err) + err = ServeHTTP(w, req) + assert.NoError(t, err) + resp := w.Result() + body, _ := io.ReadAll(resp.Body) + assert.Equal(t, expected, string(body)) } diff --git a/phpthread.go b/phpthread.go index 5ee1ff34a..55e96a6de 100644 --- a/phpthread.go +++ b/phpthread.go @@ -5,9 +5,8 @@ import "C" import ( "net/http" "runtime" + "sync" "unsafe" - - "go.uber.org/zap" ) // representation of the actual underlying PHP thread @@ -21,6 +20,7 @@ type phpThread struct { drainChan chan struct{} handler threadHandler state *threadState + mu *sync.Mutex } // interface that defines how the callbacks from the C thread should be handled @@ -30,16 +30,30 @@ type threadHandler interface { getActiveRequest() *http.Request } +func newPHPThread(threadIndex int) *phpThread { + return &phpThread{ + threadIndex: threadIndex, + drainChan: make(chan struct{}), + requestChan: make(chan *http.Request), + mu: &sync.Mutex{}, + state: newThreadState(), + } +} + func (thread *phpThread) getActiveRequest() *http.Request { return thread.handler.getActiveRequest() } // change the thread handler safely func (thread *phpThread) setHandler(handler threadHandler) { - logger.Debug("transitioning thread", zap.Int("threadIndex", thread.threadIndex)) + thread.mu.Lock() + defer thread.mu.Unlock() + if thread.state.is(stateShuttingDown) { + return + } thread.state.set(stateTransitionRequested) close(thread.drainChan) - thread.state.waitFor(stateTransitionInProgress) + thread.state.waitFor(stateTransitionInProgress, stateShuttingDown) thread.handler = handler thread.drainChan = make(chan struct{}) thread.state.set(stateTransitionComplete) diff --git a/testdata/sleep.php b/testdata/sleep.php deleted file mode 100644 index d2c78b865..000000000 --- a/testdata/sleep.php +++ /dev/null @@ -1,4 +0,0 @@ - Date: Sat, 7 Dec 2024 18:06:34 +0100 Subject: [PATCH 48/64] Fixes the testdata path. --- phpmainthread_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/phpmainthread_test.go b/phpmainthread_test.go index 25e448f2c..85458323f 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -14,6 +14,8 @@ import ( "go.uber.org/zap" ) +var testDataPath, _ = filepath.Abs("./testdata") + func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { logger = zap.NewNop() // the logger needs to not be nil assert.NoError(t, initPHPThreads(1)) // reserve 1 thread @@ -79,8 +81,8 @@ func TestTransitionThreadsWhileDoingRequests(t *testing.T) { isRunning := atomic.Bool{} isRunning.Store(true) wg := sync.WaitGroup{} - worker1Path, _ := filepath.Abs("./testdata/transition-worker-1.php") - worker2Path, _ := filepath.Abs("./testdata/transition-worker-2.php") + worker1Path := testDataPath + "/transition-worker-1.php" + worker2Path := testDataPath + "/transition-worker-2.php" Init( WithNumThreads(numThreads), @@ -136,9 +138,8 @@ func getDummyWorker(fileName string) *worker { if workers == nil { workers = make(map[string]*worker) } - absFileName, _ := filepath.Abs("./testdata/" + fileName) worker, _ := newWorker(workerOpt{ - fileName: absFileName, + fileName: testDataPath + "/" + fileName, num: 1, }) return worker @@ -147,7 +148,8 @@ func getDummyWorker(fileName string) *worker { func assertRequestBody(t *testing.T, url string, expected string) { r := httptest.NewRequest("GET", url, nil) w := httptest.NewRecorder() - req, err := NewRequestWithContext(r, WithRequestDocumentRoot("/go/src/app/testdata", false)) + + req, err := NewRequestWithContext(r, WithRequestDocumentRoot(testDataPath, false)) assert.NoError(t, err) err = ServeHTTP(w, req) assert.NoError(t, err) From 06af5d580c5662689ad130480d9d96f7ac86e768 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 18:10:52 +0100 Subject: [PATCH 49/64] Formatting. --- frankenphp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index b4cde79cd..d3780b49d 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -837,8 +837,8 @@ static void *php_thread(void *arg) { // if go signals to stop, break the loop if (scriptName == NULL) { - break; - } + break; + } int exit_status = frankenphp_execute_script(scriptName); go_frankenphp_after_script_execution(thread_index, exit_status); From 71c16bc1527fb248e520ab8269e88a1911ffe5fb Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 18:40:43 +0100 Subject: [PATCH 50/64] Allows transitioning back to inactive state. --- phpmainthread_test.go | 6 +++--- state.go | 5 +++-- thread-inactive.go | 15 +++++++++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/phpmainthread_test.go b/phpmainthread_test.go index 85458323f..d59b46235 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -42,9 +42,9 @@ func TestTransitionRegularThreadToWorkerThread(t *testing.T) { assert.IsType(t, &workerThread{}, phpThreads[0].handler) assert.Len(t, worker.threads, 1) - // transition back to regular thread - convertToRegularThread(phpThreads[0]) - assert.IsType(t, ®ularThread{}, phpThreads[0].handler) + // transition back to inactive thread + convertToInactiveThread(phpThreads[0]) + assert.IsType(t, &inactiveThread{}, phpThreads[0].handler) assert.Len(t, worker.threads, 0) drainPHPThreads() diff --git a/state.go b/state.go index 28a9085ea..4f881b0dc 100644 --- a/state.go +++ b/state.go @@ -9,7 +9,7 @@ import ( type stateID int const ( - // initial state + // livecycle states of a thread stateBooting stateID = iota stateInactive stateReady @@ -20,7 +20,7 @@ const ( stateRestarting stateYielding - // states necessary for transitioning + // states necessary for transitioning between different handlers stateTransitionRequested stateTransitionInProgress stateTransitionComplete @@ -89,6 +89,7 @@ func (h *threadState) set(nextState stateID) { } func (ts *threadState) name() string { + // TODO: return the actual name for logging/metrics return "state:" + strconv.Itoa(int(ts.get())) } diff --git a/thread-inactive.go b/thread-inactive.go index 311ecabed..c2e552262 100644 --- a/thread-inactive.go +++ b/thread-inactive.go @@ -11,7 +11,11 @@ type inactiveThread struct { } func convertToInactiveThread(thread *phpThread) { - thread.handler = &inactiveThread{thread: thread} + if thread.handler == nil { + thread.handler = &inactiveThread{thread: thread} + return + } + thread.setHandler(&inactiveThread{thread: thread}) } func (thread *inactiveThread) getActiveRequest() *http.Request { @@ -20,16 +24,19 @@ func (thread *inactiveThread) getActiveRequest() *http.Request { func (handler *inactiveThread) beforeScriptExecution() string { thread := handler.thread - thread.state.set(stateInactive) - // wait for external signal to start or shut down - thread.state.waitFor(stateTransitionRequested, stateShuttingDown) switch thread.state.get() { case stateTransitionRequested: thread.state.set(stateTransitionInProgress) thread.state.waitFor(stateTransitionComplete, stateShuttingDown) // execute beforeScriptExecution of the new handler return thread.handler.beforeScriptExecution() + case stateBooting, stateTransitionComplete: + // TODO: there's a tiny race condition here between checking and setting + thread.state.set(stateInactive) + // wait for external signal to start or shut down + thread.state.waitFor(stateTransitionRequested, stateShuttingDown) + return handler.beforeScriptExecution() case stateShuttingDown: // signal to stop return "" From 5095342a2b35518226911155725a7181c4288be2 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 19:19:29 +0100 Subject: [PATCH 51/64] Fixes go linting. --- phpmainthread_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/phpmainthread_test.go b/phpmainthread_test.go index d59b46235..d826366ee 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -84,24 +84,26 @@ func TestTransitionThreadsWhileDoingRequests(t *testing.T) { worker1Path := testDataPath + "/transition-worker-1.php" worker2Path := testDataPath + "/transition-worker-2.php" - Init( + assert.NoError(t, Init( WithNumThreads(numThreads), - WithWorkers(worker1Path, 4, map[string]string{"ENV1": "foo"}, []string{}), - WithWorkers(worker2Path, 4, map[string]string{"ENV1": "foo"}, []string{}), + WithWorkers(worker1Path, 1, map[string]string{"ENV1": "foo"}, []string{}), + WithWorkers(worker2Path, 1, map[string]string{"ENV1": "foo"}, []string{}), WithLogger(zap.NewNop()), - ) + )) - // randomly transition threads between regular and 2 worker threads + // randomly transition threads between regular, inactive and 2 worker threads go func() { for { for i := 0; i < numThreads; i++ { - switch rand.IntN(3) { + switch rand.IntN(4) { case 0: convertToRegularThread(phpThreads[i]) case 1: convertToWorkerThread(phpThreads[i], workers[worker1Path]) case 2: convertToWorkerThread(phpThreads[i], workers[worker2Path]) + case 3: + convertToInactiveThread(phpThreads[i]) } time.Sleep(time.Millisecond) if !isRunning.Load() { From 4b1805939418e87ed365ec7fe8c00eb79030be9a Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 19:25:07 +0100 Subject: [PATCH 52/64] Formatting. --- phpmainthread_test.go | 2 +- phpthread.go | 8 ++++---- state.go | 10 +++++----- thread-inactive.go | 8 ++++---- thread-worker.go | 20 ++++++++++---------- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/phpmainthread_test.go b/phpmainthread_test.go index d826366ee..601218ee2 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -103,7 +103,7 @@ func TestTransitionThreadsWhileDoingRequests(t *testing.T) { case 2: convertToWorkerThread(phpThreads[i], workers[worker2Path]) case 3: - convertToInactiveThread(phpThreads[i]) + convertToInactiveThread(phpThreads[i]) } time.Sleep(time.Millisecond) if !isRunning.Load() { diff --git a/phpthread.go b/phpthread.go index 55e96a6de..6844d4cd3 100644 --- a/phpthread.go +++ b/phpthread.go @@ -40,10 +40,6 @@ func newPHPThread(threadIndex int) *phpThread { } } -func (thread *phpThread) getActiveRequest() *http.Request { - return thread.handler.getActiveRequest() -} - // change the thread handler safely func (thread *phpThread) setHandler(handler threadHandler) { thread.mu.Lock() @@ -59,6 +55,10 @@ func (thread *phpThread) setHandler(handler threadHandler) { thread.state.set(stateTransitionComplete) } +func (thread *phpThread) getActiveRequest() *http.Request { + return thread.handler.getActiveRequest() +} + // Pin a string that is not null-terminated // PHP's zend_string may contain null-bytes func (thread *phpThread) pinString(s string) *C.char { diff --git a/state.go b/state.go index 4f881b0dc..ee9951841 100644 --- a/state.go +++ b/state.go @@ -61,6 +61,11 @@ func (ts *threadState) compareAndSwap(compareTo stateID, swapTo stateID) bool { return false } +func (ts *threadState) name() string { + // TODO: return the actual name for logging/metrics + return "state:" + strconv.Itoa(int(ts.get())) +} + func (ts *threadState) get() stateID { ts.mu.RLock() defer ts.mu.RUnlock() @@ -88,11 +93,6 @@ func (h *threadState) set(nextState stateID) { h.subscribers = newSubscribers } -func (ts *threadState) name() string { - // TODO: return the actual name for logging/metrics - return "state:" + strconv.Itoa(int(ts.get())) -} - // block until the thread reaches a certain state func (h *threadState) waitFor(states ...stateID) { h.mu.Lock() diff --git a/thread-inactive.go b/thread-inactive.go index c2e552262..f648b6a2f 100644 --- a/thread-inactive.go +++ b/thread-inactive.go @@ -18,10 +18,6 @@ func convertToInactiveThread(thread *phpThread) { thread.setHandler(&inactiveThread{thread: thread}) } -func (thread *inactiveThread) getActiveRequest() *http.Request { - panic("inactive threads have no requests") -} - func (handler *inactiveThread) beforeScriptExecution() string { thread := handler.thread @@ -47,3 +43,7 @@ func (handler *inactiveThread) beforeScriptExecution() string { func (thread *inactiveThread) afterScriptExecution(exitStatus int) { panic("inactive threads should not execute scripts") } + +func (thread *inactiveThread) getActiveRequest() *http.Request { + panic("inactive threads have no requests") +} diff --git a/thread-worker.go b/thread-worker.go index 75d2433f1..d51d1b9e6 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -41,14 +41,6 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { } } -func (handler *workerThread) getActiveRequest() *http.Request { - if handler.workerRequest != nil { - return handler.workerRequest - } - - return handler.fakeRequest -} - // return the name of the script or an empty string if no script should be executed func (handler *workerThread) beforeScriptExecution() string { switch handler.state.get() { @@ -68,7 +60,7 @@ func (handler *workerThread) beforeScriptExecution() string { handler.state.waitFor(stateReady, stateShuttingDown) return handler.beforeScriptExecution() case stateReady, stateTransitionComplete: - setUpWorkerScript(handler, handler.worker) + setupWorkerScript(handler, handler.worker) return handler.worker.fileName } panic("unexpected state: " + handler.state.name()) @@ -78,7 +70,15 @@ func (handler *workerThread) afterScriptExecution(exitStatus int) { tearDownWorkerScript(handler, exitStatus) } -func setUpWorkerScript(handler *workerThread, worker *worker) { +func (handler *workerThread) getActiveRequest() *http.Request { + if handler.workerRequest != nil { + return handler.workerRequest + } + + return handler.fakeRequest +} + +func setupWorkerScript(handler *workerThread, worker *worker) { handler.backoff.wait() metrics.StartWorker(worker.fileName) From 15429d99b690ec0440593949aa652647f3c4480d Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 21:36:10 +0100 Subject: [PATCH 53/64] Removes duplication. --- phpthread.go | 10 ++++++++++ thread-inactive.go | 5 +---- thread-regular.go | 10 +++------- thread-worker.go | 13 ++++--------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/phpthread.go b/phpthread.go index 6844d4cd3..4bfcffb66 100644 --- a/phpthread.go +++ b/phpthread.go @@ -41,6 +41,7 @@ func newPHPThread(threadIndex int) *phpThread { } // change the thread handler safely +// must be called from outside of the PHP thread func (thread *phpThread) setHandler(handler threadHandler) { thread.mu.Lock() defer thread.mu.Unlock() @@ -55,6 +56,15 @@ func (thread *phpThread) setHandler(handler threadHandler) { thread.state.set(stateTransitionComplete) } +// transition to a new handler safely +// is triggered by setHandler and executed on the PHP thread +func (thread *phpThread) transitionToNewHandler() string { + thread.state.set(stateTransitionInProgress) + thread.state.waitFor(stateTransitionComplete, stateShuttingDown) + // execute beforeScriptExecution of the new handler + return thread.handler.beforeScriptExecution() +} + func (thread *phpThread) getActiveRequest() *http.Request { return thread.handler.getActiveRequest() } diff --git a/thread-inactive.go b/thread-inactive.go index f648b6a2f..d5cfdece7 100644 --- a/thread-inactive.go +++ b/thread-inactive.go @@ -23,10 +23,7 @@ func (handler *inactiveThread) beforeScriptExecution() string { switch thread.state.get() { case stateTransitionRequested: - thread.state.set(stateTransitionInProgress) - thread.state.waitFor(stateTransitionComplete, stateShuttingDown) - // execute beforeScriptExecution of the new handler - return thread.handler.beforeScriptExecution() + return thread.transitionToNewHandler() case stateBooting, stateTransitionComplete: // TODO: there's a tiny race condition here between checking and setting thread.state.set(stateInactive) diff --git a/thread-regular.go b/thread-regular.go index ee9839d2c..6b9dd9569 100644 --- a/thread-regular.go +++ b/thread-regular.go @@ -26,19 +26,15 @@ func convertToRegularThread(thread *phpThread) { func (handler *regularThread) beforeScriptExecution() string { switch handler.state.get() { case stateTransitionRequested: - thread := handler.thread - thread.state.set(stateTransitionInProgress) - thread.state.waitFor(stateTransitionComplete, stateShuttingDown) - // execute beforeScriptExecution of the new handler - return thread.handler.beforeScriptExecution() + return handler.thread.transitionToNewHandler() case stateTransitionComplete: handler.state.set(stateReady) return handler.waitForRequest() + case stateReady: + return handler.waitForRequest() case stateShuttingDown: // signal to stop return "" - case stateReady: - return handler.waitForRequest() } panic("unexpected state: " + handler.state.name()) } diff --git a/thread-worker.go b/thread-worker.go index d51d1b9e6..bf0f7a2e2 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -45,16 +45,8 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { func (handler *workerThread) beforeScriptExecution() string { switch handler.state.get() { case stateTransitionRequested: - thread := handler.thread handler.worker.detachThread(handler.thread) - thread.state.set(stateTransitionInProgress) - thread.state.waitFor(stateTransitionComplete, stateShuttingDown) - - // execute beforeScriptExecution of the new handler - return thread.handler.beforeScriptExecution() - case stateShuttingDown: - // signal to stop - return "" + return handler.thread.transitionToNewHandler() case stateRestarting: handler.state.set(stateYielding) handler.state.waitFor(stateReady, stateShuttingDown) @@ -62,6 +54,9 @@ func (handler *workerThread) beforeScriptExecution() string { case stateReady, stateTransitionComplete: setupWorkerScript(handler, handler.worker) return handler.worker.fileName + case stateShuttingDown: + // signal to stop + return "" } panic("unexpected state: " + handler.state.name()) } From c080608661025a79913b92d453b118a790c9bf7b Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 21:51:29 +0100 Subject: [PATCH 54/64] Applies suggestions by @dunglas --- frankenphp.c | 2 ++ frankenphp.h | 1 + frankenphp_arginfo.h | 2 +- phpmainthread.go | 7 ++++--- testdata/transition-regular.php | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index d3780b49d..9a5d029de 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -823,6 +823,7 @@ static void *php_thread(void *arg) { ZEND_TSRMLS_CACHE_UPDATE(); #endif #endif + local_ctx = malloc(sizeof(frankenphp_server_context)); /* check if a default filter is set in php.ini and only filter if @@ -928,6 +929,7 @@ int frankenphp_new_main_thread(int num_threads) { 0) { return -1; } + return pthread_detach(thread); } diff --git a/frankenphp.h b/frankenphp.h index 2ed926d96..5e498b6c7 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -53,6 +53,7 @@ int frankenphp_request_startup(); int frankenphp_execute_script(char *file_name); int frankenphp_execute_script_cli(char *script, int argc, char **argv); + int frankenphp_execute_php_function(const char *php_function); void frankenphp_register_variables_from_request_info( diff --git a/frankenphp_arginfo.h b/frankenphp_arginfo.h index cecffd88d..c1bd7b550 100644 --- a/frankenphp_arginfo.h +++ b/frankenphp_arginfo.h @@ -49,4 +49,4 @@ static const zend_function_entry ext_functions[] = { ZEND_FALIAS(apache_response_headers, frankenphp_response_headers, arginfo_apache_response_headers) ZEND_FE_END }; -// clang-format on \ No newline at end of file +// clang-format on diff --git a/phpmainthread.go b/phpmainthread.go index c0ffb1614..3039ae064 100644 --- a/phpmainthread.go +++ b/phpmainthread.go @@ -3,8 +3,9 @@ package frankenphp // #include "frankenphp.h" import "C" import ( - "fmt" "sync" + + "go.uber.org/zap" ) // represents the main PHP thread @@ -20,7 +21,7 @@ var ( mainThread *phpMainThread ) -// reserve a fixed number of PHP threads on the go side +// reserve a fixed number of PHP threads on the Go side func initPHPThreads(numThreads int) error { mainThread = &phpMainThread{ state: newThreadState(), @@ -45,7 +46,7 @@ func initPHPThreads(numThreads int) error { for _, thread := range phpThreads { go func() { if !C.frankenphp_new_php_thread(C.uintptr_t(thread.threadIndex)) { - panic(fmt.Sprintf("unable to create thread %d", thread.threadIndex)) + logger.Panic("unable to create thread", zap.Int("threadIndex", thread.threadIndex)) } thread.state.waitFor(stateInactive) ready.Done() diff --git a/testdata/transition-regular.php b/testdata/transition-regular.php index c6f3efa95..31c7f436c 100644 --- a/testdata/transition-regular.php +++ b/testdata/transition-regular.php @@ -1,3 +1,3 @@ Date: Sat, 7 Dec 2024 21:57:50 +0100 Subject: [PATCH 55/64] Removes redundant check. --- thread-worker.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/thread-worker.go b/thread-worker.go index bf0f7a2e2..620cfc676 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -36,9 +36,6 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { }, }) worker.attachThread(thread) - if worker.fileName == "" { - panic("worker script is empty") - } } // return the name of the script or an empty string if no script should be executed From 9491e6b25d2a0026b9cca2e1ca98e928098b23fe Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sat, 7 Dec 2024 22:08:22 +0100 Subject: [PATCH 56/64] Locks the handler on restart. --- phpmainthread.go | 4 ++-- phpthread.go | 8 ++++---- worker.go | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/phpmainthread.go b/phpmainthread.go index 3039ae064..d8376883c 100644 --- a/phpmainthread.go +++ b/phpmainthread.go @@ -61,7 +61,7 @@ func drainPHPThreads() { doneWG := sync.WaitGroup{} doneWG.Add(len(phpThreads)) for _, thread := range phpThreads { - thread.mu.Lock() + thread.handlerMu.Lock() thread.state.set(stateShuttingDown) close(thread.drainChan) } @@ -69,7 +69,7 @@ func drainPHPThreads() { for _, thread := range phpThreads { go func(thread *phpThread) { thread.state.waitFor(stateDone) - thread.mu.Unlock() + thread.handlerMu.Unlock() doneWG.Done() }(thread) } diff --git a/phpthread.go b/phpthread.go index 4bfcffb66..f94b498fe 100644 --- a/phpthread.go +++ b/phpthread.go @@ -18,9 +18,9 @@ type phpThread struct { knownVariableKeys map[string]*C.zend_string requestChan chan *http.Request drainChan chan struct{} + handlerMu *sync.Mutex handler threadHandler state *threadState - mu *sync.Mutex } // interface that defines how the callbacks from the C thread should be handled @@ -35,7 +35,7 @@ func newPHPThread(threadIndex int) *phpThread { threadIndex: threadIndex, drainChan: make(chan struct{}), requestChan: make(chan *http.Request), - mu: &sync.Mutex{}, + handlerMu: &sync.Mutex{}, state: newThreadState(), } } @@ -43,8 +43,8 @@ func newPHPThread(threadIndex int) *phpThread { // change the thread handler safely // must be called from outside of the PHP thread func (thread *phpThread) setHandler(handler threadHandler) { - thread.mu.Lock() - defer thread.mu.Unlock() + thread.handlerMu.Lock() + defer thread.handlerMu.Unlock() if thread.state.is(stateShuttingDown) { return } diff --git a/worker.go b/worker.go index 374361591..49ddcc3be 100644 --- a/worker.go +++ b/worker.go @@ -87,6 +87,7 @@ func restartWorkers() { worker.threadMutex.RLock() ready.Add(len(worker.threads)) for _, thread := range worker.threads { + thread.handlerMu.Lock() thread.state.set(stateRestarting) close(thread.drainChan) go func(thread *phpThread) { @@ -100,6 +101,7 @@ func restartWorkers() { for _, thread := range worker.threads { thread.drainChan = make(chan struct{}) thread.state.set(stateReady) + thread.handlerMu.Unlock() } worker.threadMutex.RUnlock() } From e795c86933cbda3fd23f16d96d918a673619b12c Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Sun, 8 Dec 2024 01:01:26 +0100 Subject: [PATCH 57/64] Removes unnecessary log. --- thread-worker.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/thread-worker.go b/thread-worker.go index 620cfc676..2f1f8d8fc 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -100,8 +100,6 @@ func setupWorkerScript(handler *workerThread, worker *worker) { } func tearDownWorkerScript(handler *workerThread, exitStatus int) { - - logger.Info("tear down worker script") // if the worker request is not nil, the script might have crashed // make sure to close the worker request context if handler.workerRequest != nil { From ef1bd0d97546531e5584adb1c67bdb92841e4f97 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Mon, 9 Dec 2024 18:58:49 +0100 Subject: [PATCH 58/64] Changes Unpin() logic as suggested by @withinboredom --- cgi.go | 2 -- frankenphp.c | 2 +- phpthread.go | 3 +++ thread-regular.go | 1 - thread-worker.go | 5 +++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cgi.go b/cgi.go index e9bb736ad..b41638762 100644 --- a/cgi.go +++ b/cgi.go @@ -227,8 +227,6 @@ func go_frankenphp_release_known_variable_keys(threadIndex C.uintptr_t) { for _, v := range thread.knownVariableKeys { C.frankenphp_release_zend_string(v) } - // release everything that might still be pinned to the thread - thread.Unpin() thread.knownVariableKeys = nil } diff --git a/frankenphp.c b/frankenphp.c index 9a5d029de..c2e4f10d9 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -89,7 +89,7 @@ static void frankenphp_free_request_context() { free(ctx->cookie_data); ctx->cookie_data = NULL; - /* Is freed via thread.Unpin() at the end of each request */ + /* Is freed via thread.Unpin() */ SG(request_info).auth_password = NULL; SG(request_info).auth_user = NULL; SG(request_info).request_method = NULL; diff --git a/phpthread.go b/phpthread.go index f94b498fe..edce7fbe5 100644 --- a/phpthread.go +++ b/phpthread.go @@ -102,10 +102,13 @@ func go_frankenphp_after_script_execution(threadIndex C.uintptr_t, exitStatus C. panic(ScriptExecutionError) } thread.handler.afterScriptExecution(int(exitStatus)) + + // unpin all memory used during script execution thread.Unpin() } //export go_frankenphp_on_thread_shutdown func go_frankenphp_on_thread_shutdown(threadIndex C.uintptr_t) { + phpThreads[threadIndex].Unpin() phpThreads[threadIndex].state.set(stateDone) } diff --git a/thread-regular.go b/thread-regular.go index 6b9dd9569..b08d40682 100644 --- a/thread-regular.go +++ b/thread-regular.go @@ -61,7 +61,6 @@ func (handler *regularThread) waitForRequest() string { if err := updateServerContext(handler.thread, r, true, false); err != nil { rejectRequest(fc.responseWriter, err.Error()) handler.afterRequest(0) - handler.thread.Unpin() // go back to beforeScriptExecution return handler.beforeScriptExecution() } diff --git a/thread-worker.go b/thread-worker.go index 2f1f8d8fc..d96c07b63 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -140,6 +140,9 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { } func (handler *workerThread) waitForWorkerRequest() bool { + // unpin any memory left over from previous requests + handler.thread.Unpin() + if c := logger.Check(zapcore.DebugLevel, "waiting for request"); c != nil { c.Write(zap.String("worker", handler.worker.fileName)) } @@ -180,7 +183,6 @@ func (handler *workerThread) waitForWorkerRequest() bool { rejectRequest(fc.responseWriter, err.Error()) maybeCloseContext(fc) handler.workerRequest = nil - handler.thread.Unpin() return handler.waitForWorkerRequest() } @@ -201,7 +203,6 @@ func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t) { maybeCloseContext(fc) thread.handler.(*workerThread).workerRequest = nil - thread.Unpin() if c := fc.logger.Check(zapcore.DebugLevel, "request handling finished"); c != nil { c.Write(zap.String("worker", fc.scriptFilename), zap.String("url", r.RequestURI)) From a8a454504e7cf5763df2f0c17caea8ae17dcf7ff Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 10 Dec 2024 21:55:03 +0100 Subject: [PATCH 59/64] Adds suggestions by @dunglas and resolves TODO. --- frankenphp.c | 16 +++------ phpmainthread.go | 2 +- phpmainthread_test.go | 6 ++-- phpthread.go | 9 ++--- state.go | 84 +++++++++++++++++++++++++++++-------------- state_test.go | 14 ++++++-- thread-inactive.go | 1 - thread-regular.go | 4 +-- thread-worker.go | 5 ++- worker.go | 12 ++++--- 10 files changed, 94 insertions(+), 59 deletions(-) diff --git a/frankenphp.c b/frankenphp.c index c2e4f10d9..e0e5095c4 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -832,17 +832,11 @@ static void *php_thread(void *arg) { cfg_get_string("filter.default", &default_filter); should_filter_var = default_filter != NULL; - // perform work until go signals to stop - while (true) { - char *scriptName = go_frankenphp_before_script_execution(thread_index); - - // if go signals to stop, break the loop - if (scriptName == NULL) { - break; - } - - int exit_status = frankenphp_execute_script(scriptName); - go_frankenphp_after_script_execution(thread_index, exit_status); + // loop until Go signals to stop + char *scriptName = NULL; + while ((scriptName = go_frankenphp_before_script_execution(thread_index))) { + go_frankenphp_after_script_execution(thread_index, + frankenphp_execute_script(scriptName)); } go_frankenphp_release_known_variable_keys(thread_index); diff --git a/phpmainthread.go b/phpmainthread.go index d8376883c..5561cbd77 100644 --- a/phpmainthread.go +++ b/phpmainthread.go @@ -62,7 +62,7 @@ func drainPHPThreads() { doneWG.Add(len(phpThreads)) for _, thread := range phpThreads { thread.handlerMu.Lock() - thread.state.set(stateShuttingDown) + _ = thread.state.requestSafeStateChange(stateShuttingDown) close(thread.drainChan) } close(mainThread.done) diff --git a/phpmainthread_test.go b/phpmainthread_test.go index 601218ee2..6d0cf0f60 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -37,7 +37,7 @@ func TestTransitionRegularThreadToWorkerThread(t *testing.T) { assert.IsType(t, ®ularThread{}, phpThreads[0].handler) // transition to worker thread - worker := getDummyWorker("worker-transition-1.php") + worker := getDummyWorker("transition-worker-1.php") convertToWorkerThread(phpThreads[0], worker) assert.IsType(t, &workerThread{}, phpThreads[0].handler) assert.Len(t, worker.threads, 1) @@ -54,8 +54,8 @@ func TestTransitionRegularThreadToWorkerThread(t *testing.T) { func TestTransitionAThreadBetween2DifferentWorkers(t *testing.T) { logger = zap.NewNop() assert.NoError(t, initPHPThreads(1)) - firstWorker := getDummyWorker("worker-transition-1.php") - secondWorker := getDummyWorker("worker-transition-2.php") + firstWorker := getDummyWorker("transition-worker-1.php") + secondWorker := getDummyWorker("transition-worker-2.php") // convert to first worker thread convertToWorkerThread(phpThreads[0], firstWorker) diff --git a/phpthread.go b/phpthread.go index edce7fbe5..eabc58a98 100644 --- a/phpthread.go +++ b/phpthread.go @@ -43,14 +43,15 @@ func newPHPThread(threadIndex int) *phpThread { // change the thread handler safely // must be called from outside of the PHP thread func (thread *phpThread) setHandler(handler threadHandler) { + logger.Debug("setHandler") thread.handlerMu.Lock() defer thread.handlerMu.Unlock() - if thread.state.is(stateShuttingDown) { + if !thread.state.requestSafeStateChange(stateTransitionRequested) { + // no state change allowed == shutdown return } - thread.state.set(stateTransitionRequested) close(thread.drainChan) - thread.state.waitFor(stateTransitionInProgress, stateShuttingDown) + thread.state.waitFor(stateTransitionInProgress) thread.handler = handler thread.drainChan = make(chan struct{}) thread.state.set(stateTransitionComplete) @@ -60,7 +61,7 @@ func (thread *phpThread) setHandler(handler threadHandler) { // is triggered by setHandler and executed on the PHP thread func (thread *phpThread) transitionToNewHandler() string { thread.state.set(stateTransitionInProgress) - thread.state.waitFor(stateTransitionComplete, stateShuttingDown) + thread.state.waitFor(stateTransitionComplete) // execute beforeScriptExecution of the new handler return thread.handler.beforeScriptExecution() } diff --git a/state.go b/state.go index ee9951841..05d9e8e65 100644 --- a/state.go +++ b/state.go @@ -6,16 +6,18 @@ import ( "sync" ) -type stateID int +type stateID uint8 const ( - // livecycle states of a thread + // lifecycle states of a thread stateBooting stateID = iota - stateInactive - stateReady stateShuttingDown stateDone + // these states are safe to transition from at any time + stateInactive + stateReady + // states necessary for restarting workers stateRestarting stateYielding @@ -47,18 +49,22 @@ func newThreadState() *threadState { func (ts *threadState) is(state stateID) bool { ts.mu.RLock() - defer ts.mu.RUnlock() - return ts.currentState == state + ok := ts.currentState == state + ts.mu.RUnlock() + + return ok } func (ts *threadState) compareAndSwap(compareTo stateID, swapTo stateID) bool { ts.mu.Lock() - defer ts.mu.Unlock() - if ts.currentState == compareTo { + ok := ts.currentState == compareTo + if ok { ts.currentState = swapTo - return true + ts.notifySubscribers(swapTo) } - return false + ts.mu.Unlock() + + return ok } func (ts *threadState) name() string { @@ -68,43 +74,69 @@ func (ts *threadState) name() string { func (ts *threadState) get() stateID { ts.mu.RLock() - defer ts.mu.RUnlock() - return ts.currentState + id := ts.currentState + ts.mu.RUnlock() + + return id } -func (h *threadState) set(nextState stateID) { - h.mu.Lock() - defer h.mu.Unlock() - h.currentState = nextState +func (ts *threadState) set(nextState stateID) { + ts.mu.Lock() + ts.currentState = nextState + ts.notifySubscribers(nextState) + ts.mu.Unlock() +} - if len(h.subscribers) == 0 { +func (ts *threadState) notifySubscribers(nextState stateID) { + if len(ts.subscribers) == 0 { return } - newSubscribers := []stateSubscriber{} // notify subscribers to the state change - for _, sub := range h.subscribers { + for _, sub := range ts.subscribers { if !slices.Contains(sub.states, nextState) { newSubscribers = append(newSubscribers, sub) continue } close(sub.ch) } - h.subscribers = newSubscribers + ts.subscribers = newSubscribers } // block until the thread reaches a certain state -func (h *threadState) waitFor(states ...stateID) { - h.mu.Lock() - if slices.Contains(states, h.currentState) { - h.mu.Unlock() +func (ts *threadState) waitFor(states ...stateID) { + ts.mu.Lock() + if slices.Contains(states, ts.currentState) { + ts.mu.Unlock() return } sub := stateSubscriber{ states: states, ch: make(chan struct{}), } - h.subscribers = append(h.subscribers, sub) - h.mu.Unlock() + ts.subscribers = append(ts.subscribers, sub) + ts.mu.Unlock() <-sub.ch } + +// safely request a state change from a different goroutine +func (ts *threadState) requestSafeStateChange(nextState stateID) bool { + ts.mu.Lock() + switch ts.currentState { + // disallow state changes if shutting down + case stateShuttingDown: + ts.mu.Unlock() + return false + // ready and inactive are safe states to transition from + case stateReady, stateInactive: + ts.currentState = nextState + ts.notifySubscribers(nextState) + ts.mu.Unlock() + return true + } + ts.mu.Unlock() + + // wait for the state to change to a safe state + ts.waitFor(stateReady, stateInactive, stateShuttingDown) + return ts.requestSafeStateChange(nextState) +} diff --git a/state_test.go b/state_test.go index 47b68d410..0a9143c2e 100644 --- a/state_test.go +++ b/state_test.go @@ -29,19 +29,27 @@ func TestStateShouldHaveCorrectAmountOfSubscribers(t *testing.T) { go threadState.waitFor(stateInactive, stateShuttingDown) go threadState.waitFor(stateShuttingDown) - time.Sleep(1 * time.Millisecond) assertNumberOfSubscribers(t, threadState, 3) threadState.set(stateInactive) - time.Sleep(1 * time.Millisecond) assertNumberOfSubscribers(t, threadState, 1) threadState.set(stateShuttingDown) - time.Sleep(1 * time.Millisecond) assertNumberOfSubscribers(t, threadState, 0) } func assertNumberOfSubscribers(t *testing.T, threadState *threadState, expected int) { + maxWaits := 10_000 // wait for 1 second max + + for i := 0; i < maxWaits; i++ { + time.Sleep(100 * time.Microsecond) + threadState.mu.RLock() + if len(threadState.subscribers) == expected { + threadState.mu.RUnlock() + break + } + threadState.mu.RUnlock() + } threadState.mu.RLock() assert.Len(t, threadState.subscribers, expected) threadState.mu.RUnlock() diff --git a/thread-inactive.go b/thread-inactive.go index d5cfdece7..7c4810c71 100644 --- a/thread-inactive.go +++ b/thread-inactive.go @@ -25,7 +25,6 @@ func (handler *inactiveThread) beforeScriptExecution() string { case stateTransitionRequested: return thread.transitionToNewHandler() case stateBooting, stateTransitionComplete: - // TODO: there's a tiny race condition here between checking and setting thread.state.set(stateInactive) // wait for external signal to start or shut down thread.state.waitFor(stateTransitionRequested, stateShuttingDown) diff --git a/thread-regular.go b/thread-regular.go index b08d40682..88d72106b 100644 --- a/thread-regular.go +++ b/thread-regular.go @@ -22,7 +22,7 @@ func convertToRegularThread(thread *phpThread) { }) } -// return the name of the script or an empty string if no script should be executed +// beforeScriptExecution returns the name of the script or an empty string on shutdown func (handler *regularThread) beforeScriptExecution() string { switch handler.state.get() { case stateTransitionRequested: @@ -65,7 +65,7 @@ func (handler *regularThread) waitForRequest() string { return handler.beforeScriptExecution() } - // set the scriptName that should be executed + // set the scriptFilename that should be executed return fc.scriptFilename } } diff --git a/thread-worker.go b/thread-worker.go index d96c07b63..09f837d80 100644 --- a/thread-worker.go +++ b/thread-worker.go @@ -3,7 +3,6 @@ package frankenphp // #include "frankenphp.h" import "C" import ( - "fmt" "net/http" "path/filepath" "time" @@ -38,7 +37,7 @@ func convertToWorkerThread(thread *phpThread, worker *worker) { worker.attachThread(thread) } -// return the name of the script or an empty string if no script should be executed +// beforeScriptExecution returns the name of the script or an empty string on shutdown func (handler *workerThread) beforeScriptExecution() string { switch handler.state.get() { case stateTransitionRequested: @@ -133,7 +132,7 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { metrics.StopWorker(worker.fileName, StopReasonCrash) if handler.backoff.recordFailure() { if !watcherIsEnabled { - panic(fmt.Errorf("workers %q: too many consecutive failures", worker.fileName)) + logger.Panic("too many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", handler.backoff.failureCount)) } logger.Warn("many consecutive worker failures", zap.String("worker", worker.fileName), zap.Int("failures", handler.backoff.failureCount)) } diff --git a/worker.go b/worker.go index 49ddcc3be..74d30ac2c 100644 --- a/worker.go +++ b/worker.go @@ -87,8 +87,10 @@ func restartWorkers() { worker.threadMutex.RLock() ready.Add(len(worker.threads)) for _, thread := range worker.threads { - thread.handlerMu.Lock() - thread.state.set(stateRestarting) + if !thread.state.requestSafeStateChange(stateRestarting) { + // no state change allowed = shutdown + continue + } close(thread.drainChan) go func(thread *phpThread) { thread.state.waitFor(stateYielding) @@ -99,9 +101,9 @@ func restartWorkers() { ready.Wait() for _, worker := range workers { for _, thread := range worker.threads { - thread.drainChan = make(chan struct{}) - thread.state.set(stateReady) - thread.handlerMu.Unlock() + if thread.state.compareAndSwap(stateYielding, stateReady) { + thread.drainChan = make(chan struct{}) + } } worker.threadMutex.RUnlock() } From 23a63622356e9e83a441b16133ec9ef72080fab6 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 10 Dec 2024 22:23:53 +0100 Subject: [PATCH 60/64] Makes restarts fully safe. --- state.go | 2 +- worker.go | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/state.go b/state.go index 05d9e8e65..001213282 100644 --- a/state.go +++ b/state.go @@ -124,7 +124,7 @@ func (ts *threadState) requestSafeStateChange(nextState stateID) bool { ts.mu.Lock() switch ts.currentState { // disallow state changes if shutting down - case stateShuttingDown: + case stateShuttingDown, stateDone: ts.mu.Unlock() return false // ready and inactive are safe states to transition from diff --git a/worker.go b/worker.go index 74d30ac2c..1c1fc950f 100644 --- a/worker.go +++ b/worker.go @@ -83,6 +83,7 @@ func drainWorkers() { func restartWorkers() { ready := sync.WaitGroup{} + threadsToRestart := make([]*phpThread, 0) for _, worker := range workers { worker.threadMutex.RLock() ready.Add(len(worker.threads)) @@ -92,20 +93,20 @@ func restartWorkers() { continue } close(thread.drainChan) + threadsToRestart = append(threadsToRestart, thread) go func(thread *phpThread) { thread.state.waitFor(stateYielding) ready.Done() }(thread) } + worker.threadMutex.RUnlock() } + ready.Wait() - for _, worker := range workers { - for _, thread := range worker.threads { - if thread.state.compareAndSwap(stateYielding, stateReady) { - thread.drainChan = make(chan struct{}) - } - } - worker.threadMutex.RUnlock() + + for _, thread := range threadsToRestart { + thread.drainChan = make(chan struct{}) + thread.state.set(stateReady) } } From 18e3e587d83d4c4109423ef5b448df5e14afa78a Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 10 Dec 2024 23:13:39 +0100 Subject: [PATCH 61/64] Will make the initial startup fail even if the watcher is enabled (as is currently the case) --- worker.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/worker.go b/worker.go index 1c1fc950f..803e527f9 100644 --- a/worker.go +++ b/worker.go @@ -29,25 +29,33 @@ var ( func initWorkers(opt []workerOpt) error { workers = make(map[string]*worker, len(opt)) - directoriesToWatch := getDirectoriesToWatch(opt) - watcherIsEnabled = len(directoriesToWatch) > 0 + workersReady := sync.WaitGroup{} for _, o := range opt { worker, err := newWorker(o) worker.threads = make([]*phpThread, 0, o.num) + workersReady.Add(o.num) if err != nil { return err } for i := 0; i < worker.num; i++ { thread := getInactivePHPThread() convertToWorkerThread(thread, worker) + go func() { + thread.state.waitFor(stateReady) + workersReady.Done() + }() } } - if !watcherIsEnabled { + workersReady.Wait() + + directoriesToWatch := getDirectoriesToWatch(opt) + if len(directoriesToWatch) == 0 { return nil } + watcherIsEnabled = true if err := watcher.InitWatcher(directoriesToWatch, restartWorkers, getLogger()); err != nil { return err } From 3672c60fa04ff5ec797b46df72252f464bbcbcc8 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 10 Dec 2024 23:14:03 +0100 Subject: [PATCH 62/64] Also adds compareAndSwap to the test. --- state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state_test.go b/state_test.go index 0a9143c2e..29a10c348 100644 --- a/state_test.go +++ b/state_test.go @@ -34,7 +34,7 @@ func TestStateShouldHaveCorrectAmountOfSubscribers(t *testing.T) { threadState.set(stateInactive) assertNumberOfSubscribers(t, threadState, 1) - threadState.set(stateShuttingDown) + assert.True(t, threadState.compareAndSwap(stateInactive, stateShuttingDown)) assertNumberOfSubscribers(t, threadState, 0) } From 38f87b7b7b1239b0b55f8111e249ee84661ea474 Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 10 Dec 2024 23:15:48 +0100 Subject: [PATCH 63/64] Adds comment. --- testdata/transition-worker-2.php | 1 + 1 file changed, 1 insertion(+) diff --git a/testdata/transition-worker-2.php b/testdata/transition-worker-2.php index 969c6db20..1fb7c4271 100644 --- a/testdata/transition-worker-2.php +++ b/testdata/transition-worker-2.php @@ -2,6 +2,7 @@ while (frankenphp_handle_request(function () { echo "Hello from worker 2"; + // Simulate work to force potential race conditions (phpmainthread_test.go) usleep(1000); })) { From d97ebfe161a7e6bf21ba5f3005479ed86258370d Mon Sep 17 00:00:00 2001 From: Alliballibaba Date: Tue, 10 Dec 2024 23:49:32 +0100 Subject: [PATCH 64/64] Prevents panic on initial watcher startup. --- worker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/worker.go b/worker.go index 803e527f9..bbb44c195 100644 --- a/worker.go +++ b/worker.go @@ -30,6 +30,8 @@ var ( func initWorkers(opt []workerOpt) error { workers = make(map[string]*worker, len(opt)) workersReady := sync.WaitGroup{} + directoriesToWatch := getDirectoriesToWatch(opt) + watcherIsEnabled = len(directoriesToWatch) > 0 for _, o := range opt { worker, err := newWorker(o) @@ -50,12 +52,10 @@ func initWorkers(opt []workerOpt) error { workersReady.Wait() - directoriesToWatch := getDirectoriesToWatch(opt) - if len(directoriesToWatch) == 0 { + if !watcherIsEnabled { return nil } - watcherIsEnabled = true if err := watcher.InitWatcher(directoriesToWatch, restartWorkers, getLogger()); err != nil { return err }