def build_index(full_text_IO, suffix_array_IO)
fulltext = ""
io = StringIO.new(fulltext)
io.write MAGIC
full_text_IO.write MAGIC
documents.each do |doc|
data, metadata = @doc_hash[doc]
io.write(data)
full_text_IO.write(data)
meta_txt = Marshal.dump(metadata)
footer = "\0....#{doc}\0#{meta_txt}\0"
footer[1,4] = [footer.size - 5].pack("V")
io.write(footer)
full_text_IO.write(footer)
end
scanner = StringScanner.new(fulltext)
scanner.scan(Regexp.new(Regexp.escape(MAGIC)))
count = 0
suffixes = []
until scanner.eos?
count += 1
start = scanner.pos
text = scanner.scan_until(/\0/)
suffixes.concat find_suffixes(text[0..-2], start)
len = scanner.scan(/..../).unpack("V")[0]
scanner.pos += len
end
sorted = suffixes.sort_by{|x| fulltext[x, @max_wordsize]}
sorted.each_slice(10000){|x| suffix_array_IO.write x.pack("V*")}
nil
end