remove ngram offset code (#616)

We have been running btree as the default for many months. We worried
about a performance hit, but it never happened. After some recent local
testing I did I noticed the btree actually interacted with the disk more
efficiently. So the old code both uses more memory and is slower, lets
just remove it.

Test Plan: go test ./...

btree.go

@@ -289,6 +289,7 @@ type btreeIndex struct {
 	postingIndex simpleSection
 }
 
+// SizeBytes returns how much memory this structure uses in the heap.
 func (b btreeIndex) SizeBytes() (sz int) {
 	// btree
 	if b.bt != nil {
@@ -401,6 +402,9 @@ func (b btreeIndex) getBucket(bucketIndex int) (off uint32, sz uint32) {
 	return
 }
 
+// DumpMap is a debug method which returns the btree as an in-memory
+// representation. This is how zoekt represents the ngram index in
+// google/zoekt.
 func (b btreeIndex) DumpMap() map[ngram]simpleSection {
 	if b.bt == nil {
 		return nil
@@ -427,3 +431,11 @@ func (b btreeIndex) DumpMap() map[ngram]simpleSection {
 
 	return m
 }
+
+// GetBlob returns the raw encoded offset list for ng.
+//
+// Note: the returned byte slice is mmap backed normally.
+func (b btreeIndex) GetBlob(ng ngram) ([]byte, error) {
+	sec := b.Get(ng)
+	return b.file.Read(sec.off, sec.sz)
+}

hititer.go

@@ -110,10 +110,6 @@ func (d *indexData) newDistanceTrigramIter(ng1, ng2 ngram, dist uint32, caseSens
 }
 
 func (d *indexData) trigramHitIterator(ng ngram, caseSensitive, fileName bool) (hitIterator, error) {
-	if d.ngrams == nil {
-		return nil, fmt.Errorf("trigramHitIterator: ngrams=nil")
-	}
-
 	variants := []ngram{ng}
 	if !caseSensitive {
 		variants = generateCaseNgrams(ng)

indexdata.go

@@ -33,7 +33,7 @@ type indexData struct {
 
 	file IndexFile
 
-	ngrams ngramIndex
+	ngrams btreeIndex
 
 	newlinesStart uint32
 	newlinesIndex []uint32
@@ -56,7 +56,7 @@ type indexData struct {
 
 	fileNameContent []byte
 	fileNameIndex   []uint32
-	fileNameNgrams  fileNameNgrams
+	fileNameNgrams  btreeIndex
 
 	// fileEndSymbol[i] is the index of the first symbol for document i.
 	fileEndSymbol []uint32
@@ -314,9 +314,7 @@ func (d *indexData) memoryUse() int {
 	}
 	sz += 8 * len(d.runeDocSections)
 	sz += 8 * len(d.fileBranchMasks)
-	if d.ngrams != nil {
-		sz += d.ngrams.SizeBytes()
-	}
+	sz += d.ngrams.SizeBytes()
 	sz += d.fileNameNgrams.SizeBytes()
 	return sz
 }
@@ -349,13 +347,8 @@ func lastMinarg(xs []uint32) uint32 {
 
 func (data *indexData) ngramFrequency(ng ngram, filename bool) uint32 {
 	if filename {
-		return data.fileNameNgrams.Frequency(ng)
-	}
-
-	if data.ngrams == nil {
-		return 0
+		return data.fileNameNgrams.Get(ng).sz
 	}
-
 	return data.ngrams.Get(ng).sz
 }