graph/formats/rdf: add deduplication function
MODIFY
graph/formats/rdf/urna.go
graph/formats/rdf/urna.go
@@ -16,6 +16,37 @@ import (
"gonum.org/v1/gonum/stat/combin"
)
+// Deduplicate removes duplicate statements in s, working in place, and returns
+// the deduplicated slice with statements sorted in lexical order. Term UID
+// fields are not considered and their values may be lost during deduplication.
+func Deduplicate(s []*Statement) []*Statement {
+ if len(s) < 2 {
+ return s
+ }
+ sort.Sort(c14nStatements(s))
+ curr := 0
+ for i, e := range s {
+ if isSameStatement(e, s[curr]) {
+ continue
+ }
+ curr++
+ if curr < i {
+ s[curr], s[i] = s[i], nil
+ }
+ }
+ return s[:curr+1]
+}
+
+func isSameStatement(a, b *Statement) bool {
+ if a == b {
+ return true
+ }
+ return a.Subject.Value == b.Subject.Value &&
+ a.Predicate.Value == b.Predicate.Value &&
+ a.Object.Value == b.Object.Value &&
+ a.Label.Value == b.Label.Value
+}
+
// Note on implementation details: The comment numbering in the code relates the
// implementation to the steps of the algorithm described in the specification.

MODIFY
graph/formats/rdf/urna_test.go
graph/formats/rdf/urna_test.go
@@ -12,8 +12,82 @@ import (
"path/filepath"
"strings"
"testing"
+
+ "github.com/google/go-cmp/cmp"
)
+var deduplicateTests = []struct {
+ statements string
+ want string
+}{
+ {},
+ {
+ statements: `
+_:1 <p:a> _:2 .
+`,
+ want: `_:1 <p:a> _:2 .
+`,
+ },
+ {
+ statements: `
+_:1 <p:a> _:2 .
+_:1 <p:a> _:2 .
+`,
+ want: `_:1 <p:a> _:2 .
+`,
+ },
+ {
+ statements: `
+_:1 <p:a> _:2 .
+_:2 <p:a> _:1 .
+_:1 <p:a> _:2 .
+`,
+ want: `_:1 <p:a> _:2 .
+_:2 <p:a> _:1 .
+`,
+ },
+ {
+ statements: `
+_:1 <p:a> _:2 .
+_:2 <p:a> _:1 .
+_:1 <p:a> _:2 .
+_:1 <p:a> _:2 .
+`,
+ want: `_:1 <p:a> _:2 .
+_:2 <p:a> _:1 .
+`,
+ },
+}
+
+func TestDeduplicate(t *testing.T) {
+tests:
+ for i, test := range deduplicateTests {
+ var statements []*Statement
+ dec := NewDecoder(strings.NewReader(test.statements))
+ for {
+ s, err := dec.Unmarshal()
+ if err != nil {
+ if err != io.EOF {
+ t.Errorf("error during decoding: %v", err)
+ continue tests
+ }
+ break
+ }
+ statements = append(statements, s)
+ }
+
+ var buf strings.Builder
+ for _, s := range Deduplicate(statements) {
+ fmt.Fprintln(&buf, s)
+ }
+
+ got := buf.String()
+ if got != test.want {
+ t.Errorf("unexpected result for test %d:\n%s", i, cmp.Diff(got, test.want))
+ }
+ }
+}
+
func TestURNA(t *testing.T) {
glob, err := filepath.Glob(filepath.Join("testdata", *tests))
if err != nil {

graph/formats/rdf: add deduplication function