From a91a18df3ce4efbd6b96b16bee0798ab2672cb7c Mon Sep 17 00:00:00 2001 From: nawivee <144064429+nawivee@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:50:46 +0100 Subject: [PATCH 1/6] Expose status of chunks to see which was loaded and which are not --- chunk.go | 1 + download.go | 1 + 2 files changed, 2 insertions(+) diff --git a/chunk.go b/chunk.go index f53bf86..e6511ad 100644 --- a/chunk.go +++ b/chunk.go @@ -18,4 +18,5 @@ func (dst *OffsetWriter) Write(b []byte) (n int, err error) { // Chunk represents the partial content range type Chunk struct { Start, End uint64 + IsLoaded bool } diff --git a/download.go b/download.go index 89565ae..b9d66e3 100644 --- a/download.go +++ b/download.go @@ -318,6 +318,7 @@ func (d *Download) dl(dest io.WriterAt, errC chan error) { errC <- err return } + d.chunks[i].IsLoaded = true <-max }(i) From 47bc73915ab2e6d04f92afc2f1e064c53a42cd87 Mon Sep 17 00:00:00 2001 From: nawivee <144064429+nawivee@users.noreply.github.com> Date: Mon, 11 Dec 2023 22:50:46 +0100 Subject: [PATCH 2/6] Expose status of chunks to see which was loaded and which are not --- chunk.go | 1 + chunks_test.go | 16 ++++++++-------- download.go | 11 ++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/chunk.go b/chunk.go index f53bf86..e6511ad 100644 --- a/chunk.go +++ b/chunk.go @@ -18,4 +18,5 @@ func (dst *OffsetWriter) Write(b []byte) (n int, err error) { // Chunk represents the partial content range type Chunk struct { Start, End uint64 + IsLoaded bool } diff --git a/chunks_test.go b/chunks_test.go index c3e707e..fb5c14c 100644 --- a/chunks_test.go +++ b/chunks_test.go @@ -34,23 +34,23 @@ func TestChunksLength(t *testing.T) { return } - if d.chunks[0].Start != 0 { + if d.Chunks[0].Start != 0 { - t.Errorf("First chunk should start from 0, but got %d", d.chunks[0].Start) + t.Errorf("First chunk should start from 0, but got %d", d.Chunks[0].Start) } - if chunk0.End != d.chunks[0].End { + if chunk0.End != d.Chunks[0].End { - t.Errorf("Chunk 0 expecting: %d but got: %d", chunk0.End, d.chunks[0].End) + t.Errorf("Chunk 0 expecting: %d but got: %d", chunk0.End, d.Chunks[0].End) } - if d.chunks[1].Start != 5242871 { + if d.Chunks[1].Start != 5242871 { - t.Errorf("Second chunk should start from: 5242871, but got %d", d.chunks[1].Start) + t.Errorf("Second chunk should start from: 5242871, but got %d", d.Chunks[1].Start) } - if chunk1.End != d.chunks[1].End { + if chunk1.End != d.Chunks[1].End { - t.Errorf("Chunk 1 expecting: %d but got: %d", chunk1.End, d.chunks[1].End) + t.Errorf("Chunk 1 expecting: %d but got: %d", chunk1.End, d.Chunks[1].End) } } diff --git a/download.go b/download.go index 89565ae..25d7a86 100644 --- a/download.go +++ b/download.go @@ -50,7 +50,7 @@ type ( info *Info - chunks []*Chunk + Chunks []*Chunk startedAt time.Time } @@ -156,13 +156,13 @@ func (d *Download) Init() (err error) { } chunksLen := d.info.Size / d.ChunkSize - d.chunks = make([]*Chunk, 0, chunksLen) + d.Chunks = make([]*Chunk, 0, chunksLen) // Set chunk ranges. for i := uint64(0); i < chunksLen; i++ { chunk := new(Chunk) - d.chunks = append(d.chunks, chunk) + d.Chunks = append(d.Chunks, chunk) chunk.Start = (d.ChunkSize * i) + i chunk.End = chunk.Start + d.ChunkSize @@ -305,7 +305,7 @@ func (d *Download) dl(dest io.WriterAt, errC chan error) { max = make(chan int, d.Concurrency) ) - for i := 0; i < len(d.chunks); i++ { + for i := 0; i < len(d.Chunks); i++ { max <- 1 wg.Add(1) @@ -314,10 +314,11 @@ func (d *Download) dl(dest io.WriterAt, errC chan error) { defer wg.Done() // Concurrently download and write chunk - if err := d.DownloadChunk(d.chunks[i], &OffsetWriter{dest, int64(d.chunks[i].Start)}); err != nil { + if err := d.DownloadChunk(d.Chunks[i], &OffsetWriter{dest, int64(d.Chunks[i].Start)}); err != nil { errC <- err return } + d.Chunks[i].IsLoaded = true <-max }(i) From 26f6ac7c083ad2e055526a19fa4ed2edbe0a1ea7 Mon Sep 17 00:00:00 2001 From: nawivee <144064429+nawivee@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:08:35 +0100 Subject: [PATCH 3/6] Allow to make hot download megabyte per megabyte for beginning of the file --- download.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/download.go b/download.go index 25d7a86..ab639f8 100644 --- a/download.go +++ b/download.go @@ -53,6 +53,8 @@ type ( Chunks []*Chunk startedAt time.Time + // how many megabytes to download by one megabyte before switching to bigger chunks + HotStartMegabytes uint64 } GotHeader struct { @@ -155,7 +157,33 @@ func (d *Download) Init() (err error) { d.ChunkSize = getDefaultChunkSize(d.info.Size, d.MinChunkSize, d.MaxChunkSize, uint64(d.Concurrency)) } + Megabyte := uint64(1024 * 1024) + + if d.info.Size < (1*Megabyte) && d.HotStartMegabytes > 0 { + chunk := new(Chunk) + d.Chunks = append(d.Chunks, chunk) + chunk.Start = 0 + chunk.End = d.info.Size - 1 + return nil + } + + hotStartBytes := d.HotStartMegabytes * Megabyte + chunksLen := d.info.Size / d.ChunkSize + + if d.info.Size > hotStartBytes { + + chunksLen = (d.info.Size - hotStartBytes) / d.ChunkSize + SmallChunkSize := Megabyte + for i := uint64(0); i < d.HotStartMegabytes; i++ { + chunk := new(Chunk) + d.Chunks = append(d.Chunks, chunk) + + chunk.Start = (SmallChunkSize * i) + i + chunk.End = chunk.Start + SmallChunkSize + } + } + d.Chunks = make([]*Chunk, 0, chunksLen) // Set chunk ranges. @@ -164,7 +192,7 @@ func (d *Download) Init() (err error) { chunk := new(Chunk) d.Chunks = append(d.Chunks, chunk) - chunk.Start = (d.ChunkSize * i) + i + chunk.Start = (d.ChunkSize * i) + i + hotStartBytes chunk.End = chunk.Start + d.ChunkSize if chunk.End >= d.info.Size || i == chunksLen-1 { chunk.End = d.info.Size - 1 From 559c34bc33c0b747b20c7662fd7c02103a5d6ac3 Mon Sep 17 00:00:00 2001 From: nawivee <144064429+nawivee@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:08:35 +0100 Subject: [PATCH 4/6] Allow to make hot download megabyte per megabyte for beginning of the file --- download.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/download.go b/download.go index 25d7a86..b8ccd5b 100644 --- a/download.go +++ b/download.go @@ -53,6 +53,8 @@ type ( Chunks []*Chunk startedAt time.Time + // how many megabytes to download by one megabyte before switching to bigger chunks + HotStartMegabytes uint64 } GotHeader struct { @@ -155,8 +157,34 @@ func (d *Download) Init() (err error) { d.ChunkSize = getDefaultChunkSize(d.info.Size, d.MinChunkSize, d.MaxChunkSize, uint64(d.Concurrency)) } + d.Chunks = make([]*Chunk, 0) + + Megabyte := uint64(1024 * 1024) + + if d.info.Size < (1*Megabyte) && d.HotStartMegabytes > 0 { + chunk := new(Chunk) + d.Chunks = append(d.Chunks, chunk) + chunk.Start = 0 + chunk.End = d.info.Size - 1 + return nil + } + + hotStartBytes := d.HotStartMegabytes * Megabyte + chunksLen := d.info.Size / d.ChunkSize - d.Chunks = make([]*Chunk, 0, chunksLen) + + if d.info.Size > hotStartBytes { + + chunksLen = (d.info.Size - hotStartBytes) / d.ChunkSize + SmallChunkSize := Megabyte + for i := uint64(0); i < d.HotStartMegabytes; i++ { + chunk := new(Chunk) + d.Chunks = append(d.Chunks, chunk) + + chunk.Start = (SmallChunkSize * i) + i + chunk.End = chunk.Start + SmallChunkSize + } + } // Set chunk ranges. for i := uint64(0); i < chunksLen; i++ { @@ -164,7 +192,7 @@ func (d *Download) Init() (err error) { chunk := new(Chunk) d.Chunks = append(d.Chunks, chunk) - chunk.Start = (d.ChunkSize * i) + i + chunk.Start = (d.ChunkSize * i) + i + hotStartBytes chunk.End = chunk.Start + d.ChunkSize if chunk.End >= d.info.Size || i == chunksLen-1 { chunk.End = d.info.Size - 1 From eddf0825eefa3d13817ccf3a468361e682b842d1 Mon Sep 17 00:00:00 2001 From: nawivee <144064429+nawivee@users.noreply.github.com> Date: Tue, 16 Jan 2024 16:30:43 +0100 Subject: [PATCH 5/6] Ignore mac system info --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e6ac740..26977d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ dist/ .idea +.DS_Store coverage.out *.dat *.output From 52ccb4268d40b048b8da227c938951302b4ea3bf Mon Sep 17 00:00:00 2001 From: nawivee <144064429+nawivee@users.noreply.github.com> Date: Tue, 16 Jan 2024 17:07:19 +0100 Subject: [PATCH 6/6] Allow to configure concurrency for hot start --- download.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/download.go b/download.go index b8ccd5b..7b72912 100644 --- a/download.go +++ b/download.go @@ -54,7 +54,8 @@ type ( startedAt time.Time // how many megabytes to download by one megabyte before switching to bigger chunks - HotStartMegabytes uint64 + HotStartMegabytes uint64 + HotStartConcurrency uint } GotHeader struct { @@ -151,6 +152,9 @@ func (d *Download) Init() (err error) { if d.Concurrency == 0 { d.Concurrency = getDefaultConcurrency() } + if d.HotStartConcurrency == 0 && d.HotStartMegabytes > 0 { + d.HotStartConcurrency = 2 + } // Set default chunk size if d.ChunkSize == 0 { @@ -330,10 +334,29 @@ func (d *Download) dl(dest io.WriterAt, errC chan error) { wg sync.WaitGroup // Concurrency limit. - max = make(chan int, d.Concurrency) + max = make(chan int, d.Concurrency) + hotStartMax = make(chan int, d.HotStartConcurrency) ) - for i := 0; i < len(d.Chunks); i++ { + for i := 0; i < int(d.HotStartMegabytes); i++ { + hotStartMax <- 1 + wg.Add(1) + + go func(i int) { + defer wg.Done() + + // Concurrently download and write chunk + if err := d.DownloadChunk(d.Chunks[i], &OffsetWriter{dest, int64(d.Chunks[i].Start)}); err != nil { + errC <- err + return + } + d.Chunks[i].IsLoaded = true + + <-hotStartMax + }(i) + } + + for i := int(d.HotStartMegabytes); i < len(d.Chunks); i++ { max <- 1 wg.Add(1)