From 7076c1c68a5a8524bd121d53a7d920f99d205eae Mon Sep 17 00:00:00 2001 From: Kurt Jensen Date: Fri, 16 Aug 2024 15:03:43 -0400 Subject: [PATCH] COMPUTE-15 Prefetch cache truncated read fix (#86) Modify isDataInCache() such that if there is a covered range in the cache, but the read request extends beyond the last vector in the cache, treat it as DATA_OUTSIDE_CACHE --- .github/workflows/build-packages.yml | 4 +- .github/workflows/master.yml | 10 ++--- dxfuse.go | 8 ++-- go.mod | 26 +++++++----- go.sum | 60 ++++++++++------------------ prefetch.go | 24 ++++++++--- 6 files changed, 65 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 2a7241e..02c8dd3 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.16.10' + go-version: '1.22.6' - name: Build Linux executable run: | set -x @@ -51,7 +51,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.16.10' + go-version: '1.22.6' - name: Build Macos executable run: | set -x diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d13a579..a7afe5f 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.16.10' + go-version: '1.22.6' - name: Install & build run: | @@ -33,7 +33,7 @@ jobs: python3 python3-pip python3-dev # Install printing with colors python package - python3 -m pip install --upgrade setuptools wheel pyOpenSSL + python3 -m pip install --upgrade 'setuptools<71.0.0' wheel wget https://raw.githubusercontent.com/dnanexus/dx-toolkit/master/src/python/requirements.txt python3 -m pip install -r requirements.txt @@ -63,7 +63,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.16.10' + go-version: '1.22.6' - name: Install & build run: | @@ -75,7 +75,7 @@ jobs: openssl libssl-dev zip unzip libffi-dev \ python3 python3-pip python3-dev # Install printing with colors python package - python3 -m pip install --upgrade setuptools wheel pyOpenSSL + python3 -m pip install --upgrade 'setuptools<71.0.0' wheel wget https://raw.githubusercontent.com/dnanexus/dx-toolkit/master/src/python/requirements.txt python3 -m pip install -r requirements.txt @@ -105,7 +105,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.16.10' + go-version: '1.22.6' - name: Install & build run: | set -x diff --git a/dxfuse.go b/dxfuse.go index c8579a5..3697efc 100644 --- a/dxfuse.go +++ b/dxfuse.go @@ -720,7 +720,6 @@ func (fsys *Filesys) RmDir(ctx context.Context, op *fuseops.RmDirOp) error { // // Note: We want to have a guarantied O(1) algorithm, otherwise, we would use a // randomized approach. -// func (fsys *Filesys) insertIntoFileHandleTable(fh *FileHandle) fuseops.HandleID { fsys.fhCounter++ hid := fuseops.HandleID(fsys.fhCounter) @@ -738,7 +737,6 @@ func (fsys *Filesys) insertIntoDirHandleTable(dh *DirHandle) fuseops.HandleID { } // A CreateRequest asks to create and open a file (not a directory). -// func (fsys *Filesys) CreateFile(ctx context.Context, op *fuseops.CreateFileOp) error { fsys.mutex.Lock() defer fsys.mutex.Unlock() @@ -1258,7 +1256,6 @@ func (fsys *Filesys) ReleaseDirHandle(ctx context.Context, op *fuseops.ReleaseDi // === // File handling -// func (fsys *Filesys) openRegularFile( ctx context.Context, oph *OpHandle, @@ -1317,7 +1314,6 @@ func (fsys *Filesys) openRegularFile( } // Note: What happens if the file is opened for writing? -// func (fsys *Filesys) OpenFile(ctx context.Context, op *fuseops.OpenFileOp) error { fsys.mutex.Lock() defer fsys.mutex.Unlock() @@ -1415,6 +1411,10 @@ func (fsys *Filesys) readRemoteFile(ctx context.Context, op *fuseops.ReadFileOp, // See if the data has already been prefetched. // This call will wait, if a prefetch IO is in progress. len := fsys.pgs.CacheLookup(fh.hid, op.Offset, endOfs, op.Dst) + // log received length if less than requested + if fsys.options.Verbose && int64(len) < reqSize { + fsys.log("ReadFile: CacheLookup returned %d, requested %d", len, reqSize) + } if len > 0 { op.BytesRead = len return nil diff --git a/go.mod b/go.mod index 8855973..020f3b8 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,22 @@ module github.com/dnanexus/dxfuse -go 1.16 +go 1.22.6 require ( - github.com/StackExchange/wmi v1.2.1 // indirect - github.com/dnanexus/dxda v0.5.8 - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/jacobsa/fuse v0.0.0-20211019165009-c75d3f26fceb - github.com/mattn/go-sqlite3 v1.14.9 + github.com/dnanexus/dxda v0.6.2 + github.com/jacobsa/fuse v0.0.0-20240721083925-faebccf4c6c6 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/shirou/gopsutil v3.21.11+incompatible + golang.org/x/exp v0.0.0-20200513190911-00229845015e + golang.org/x/sync v0.8.0 +) + +require ( + github.com/go-ole/go-ole v1.3.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/shirou/gopsutil v3.21.9+incompatible github.com/stretchr/testify v1.7.0 // indirect - github.com/tklauser/go-sysconf v0.3.9 // indirect - golang.org/x/exp v0.0.0-20200513190911-00229845015e - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + github.com/tklauser/go-sysconf v0.3.14 // indirect + github.com/tklauser/numcpus v0.8.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + golang.org/x/sys v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f4dd674..72edf7d 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,34 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= -github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= -github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= -github.com/dnanexus/dxda v0.5.8 h1:sz1W8WqlSoc6rp4eeZ19JniMx6RLrqX96K+Lpp/Lv6c= -github.com/dnanexus/dxda v0.5.8/go.mod h1:UeYZ7DMtJGGQ13owVEeAbUSFWXkCj97xZEzoBebZP68= +github.com/dnanexus/dxda v0.6.2 h1:qmpYDJaqd7wgbIiwdgzpCvCvRs/qBr6FCPTm2vD8pNg= +github.com/dnanexus/dxda v0.6.2/go.mod h1:NSycIpaP43ilDCoE0WXUFuPmAewkwgqEHU3JuyZY8ZQ= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/jacobsa/fuse v0.0.0-20211019165009-c75d3f26fceb h1:3azI0mTb6MzOp2/z69FJHlktZ8AI5fE04aXes4EO7yw= -github.com/jacobsa/fuse v0.0.0-20211019165009-c75d3f26fceb/go.mod h1:xtZnnLxHY6QniCrfIpTwr5h8mH8zr+jsOFj0y9cfyp4= -github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= -github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff/go.mod h1:gJWba/XXGl0UoOmBQKRWCJdHrr3nE0T65t6ioaj3mLI= -github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11/go.mod h1:+DBdDyfoO2McrOyDemRBq0q9CMEByef7sYl7JH5Q3BI= -github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb/go.mod h1:ivcmUvxXWjb27NsPEaiYK7AidlZXS7oQ5PowUS9z3I4= -github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3/go.mod h1:mPvulh9VKXvo+yOlrD4VYOOYuLdZJ36wa/5QIrtXvWs= -github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6/go.mod h1:JEWKD6V8xETMW+DEv+IQVz++f8Cn8O/X0HPeDY3qNis= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/jacobsa/fuse v0.0.0-20240721083925-faebccf4c6c6 h1:wDhMU6XRHZxgmDmR2bfWYNoG5KilEaroD2ZodIVoxFc= +github.com/jacobsa/fuse v0.0.0-20240721083925-faebccf4c6c6/go.mod h1:JYi9iIxdYNgxmMgLwtSHO/hmVnP2kfX1oc+mtx+XWLA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= -github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= -github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4/go.mod h1:RMU2gJXhratVxBDTFeOdNhd540tG57lt9FIUV0YLvIQ= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shirou/gopsutil v3.21.9+incompatible h1:LTLpUnfX81MkHeCtSrwNKZwuW5Id6kCa7/P43NdcNn4= -github.com/shirou/gopsutil v3.21.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= -github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= -github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= +github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= +github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= +github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= +github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -51,25 +39,19 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/prefetch.go b/prefetch.go index bcd8774..df8eee8 100644 --- a/prefetch.go +++ b/prefetch.go @@ -148,7 +148,7 @@ func (iov Iovec) intersectBuffer(startOfs int64, endOfs int64) []byte { bgnByte := MaxInt64(iov.startByte, startOfs) endByte := MinInt64(iov.endByte, endOfs) - // normalize, to offets inside the buffer + // normalize, to offsets inside the buffer bgnByte -= iov.startByte endByte -= iov.startByte @@ -724,6 +724,7 @@ func (pgs *PrefetchGlobalState) findCoveredRange( pfm *PrefetchFileMetadata, startOfs int64, endOfs int64) (int, int) { + // check if there is ANY intersection with cache if endOfs < pfm.cache.startByte || pfm.cache.endByte < startOfs { @@ -769,11 +770,15 @@ func (pgs *PrefetchGlobalState) findCoveredRange( // For example, if there are currently 2 chunks in cache, and the current IO falls // in the second location then: // assuming -// maxNumIovecs = 2 +// +// maxNumIovecs = 2 +// // calculate -// iovIndex = 1 -// nReadAheadChunks = iovIndex + maxNumIovecs - nIovec -// = 1 + 2 - 2 = 1 +// +// iovIndex = 1 +// nReadAheadChunks = iovIndex + maxNumIovecs - nIovec +// = 1 + 2 - 2 = 1 +// // If the IO landed on the first location, then iovIndex=0, and we will not issue // additional readahead IOs. func (pgs *PrefetchGlobalState) moveCacheWindow(pfm *PrefetchFileMetadata, iovIndex int) { @@ -951,6 +956,14 @@ func (pgs *PrefetchGlobalState) isDataInCache( // the cached area. return DATA_OUTSIDE_CACHE } + // if the end offset is greater than the last byte iov, then we are outside the cache + // If access were deemed sequential, we would have prefetched the data + if endOfs > pfm.cache.iovecs[last].endByte { + if pgs.verboseLevel >= 2 { + pfm.log("isDataInCache: endOfs=%d last=%d endByte=%d", endOfs, last, pfm.cache.iovecs[last].endByte) + } + return DATA_OUTSIDE_CACHE + } for i := first; i <= last; i++ { iov := pfm.cache.iovecs[i] @@ -1044,7 +1057,6 @@ func (pgs *PrefetchGlobalState) getDataFromCache( // This is done on behalf of a user read request. If this range has been prefetched, copy the data. // Return how much data was copied. Return zero length if the data isn't in cache. -// func (pgs *PrefetchGlobalState) CacheLookup(hid fuseops.HandleID, startOfs int64, endOfs int64, data []byte) int { pfm := pgs.getAndLockPfm(hid) if pfm == nil {