Read the published research corpus and avoid expensive paging plans [skip ci]
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCorpusTextCodecRoundTripAndOrder(t *testing.T) {
|
||||
values := []string{"", "\x00", "\x00\x01", "\x01", "\x01\x02", "\x02", "A", "A\x00", "A\x01", "Ada 张三 Ω", "Z\x00\x01\x02尾"}
|
||||
for _, value := range values {
|
||||
encoded := encodeText(value)
|
||||
decoded, err := decodeText(encoded)
|
||||
if err != nil || decoded != value || strings.ContainsRune(encoded, 0) {
|
||||
t.Fatalf("round trip failed for %q", value)
|
||||
}
|
||||
for _, other := range values {
|
||||
if strings.Compare(value, other) != strings.Compare(encoded, encodeText(other)) {
|
||||
t.Fatalf("binary ordering changed for %q %q", value, other)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, invalid := range []string{"\x01", "prefix\x01x", "\x01\x00"} {
|
||||
if _, err := decodeText(invalid); err == nil {
|
||||
t.Fatalf("accepted malformed escape %q", invalid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorpusTextColumnsRespectCodecAndNulls(t *testing.T) {
|
||||
var required string
|
||||
if err := (textColumn{target: &required}).Scan("\x01\x01"); err != nil || required != "\x01\x01" {
|
||||
t.Fatal("legacy text was decoded")
|
||||
}
|
||||
if err := (textColumn{encoded: true, target: &required}).Scan([]byte("before\x01\x01after\x01\x02")); err != nil || required != "before\x00after\x01" {
|
||||
t.Fatalf("encoded text did not decode: %q %v", required, err)
|
||||
}
|
||||
var nullable sql.NullString
|
||||
if err := (textColumn{encoded: true, target: &nullable}).Scan(nil); err != nil || nullable.Valid {
|
||||
t.Fatal("null changed into empty text")
|
||||
}
|
||||
if err := (textColumn{encoded: true, target: &nullable}).Scan("\x01\x01"); err != nil || !nullable.Valid || nullable.String != "\x00" {
|
||||
t.Fatal("nullable text was not restored")
|
||||
}
|
||||
if err := (textColumn{encoded: true, target: &required}).Scan(nil); err == nil {
|
||||
t.Fatal("accepted null for required text")
|
||||
}
|
||||
if err := (textColumn{encoded: true, target: &required}).Scan("\x01x"); err == nil {
|
||||
t.Fatal("accepted corrupt encoded text")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorpusSchemaCodecMarker(t *testing.T) {
|
||||
for _, annotation := range []sql.NullString{{}, {String: "unrelated schema description", Valid: true}} {
|
||||
encoded, err := encodedSchema(annotation)
|
||||
if encoded || err != nil {
|
||||
t.Fatal("legacy schema changed encoding")
|
||||
}
|
||||
}
|
||||
encoded, err := encodedSchema(sql.NullString{String: "paper-atlas:text-codec=nul-soh-v1", Valid: true})
|
||||
if !encoded || err != nil {
|
||||
t.Fatal("published codec was not recognized")
|
||||
}
|
||||
for _, name := range []string{"unknown", "", "legacy"} {
|
||||
if _, err := encodedSchema(sql.NullString{String: "paper-atlas:text-codec=" + name, Valid: true}); err == nil {
|
||||
t.Fatal("unknown codec was accepted")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user