Add library for mongoDB

.gitlab-ci.yml

@@ -13,7 +13,11 @@ Unit tests:
   #  paths:
   #    - "coverage.html"
   script:
-    - "go test -coverprofile c.out -v ./..."
+    - "go get gopkg.in/mgo.v2"
+    - "go test -coverprofile auth.part -v ./auth"
+    - "go test -coverprofile mgdb.part -v ./mgdb"
+    - "echo 'mode: set' *part > c.out"
+    - "grep -h -v 'mode: set' *part >> c.out"
     - "go tool cover -html=c.out -o coverage.html"
 
 ARM tests:
@@ -21,7 +25,8 @@ ARM tests:
   tags:
     - arm
   script:
-    - "go test -cover -short -v ./..."
+    - "go get gopkg.in/mgo.v2"
+    - "go test -cover -short -run 'Test[^M][^o][^n][^g][^o]' -v ./..."
 
 Code lint:
   stage: test
@@ -31,5 +36,6 @@ Code lint:
     - lint
   script:
     - "go get github.com/alecthomas/gometalinter"
+    - "go get gopkg.in/mgo.v2"
     - "gometalinter --install"
     - "gometalinter -t --deadline=60s ./..."

mgdb/README.md

+### DFSS - MGDB lib ###
+
+This library is used in order to manage a connection to mongoDB
+It uses the mgo driver, but aims at simplifying the queries to the database
+
+## Mongo Manager ##
+
+The struct handling the connection is MongoManager. It requires the environment variable MONGOHQ_URL in order to initialize a connection : it is the uri containing the informations to connect to mongo.
+For example, in a test environment, we may have :
+
+    MONGOHQ_URL=localhost (require a mongo instance running on default port 27017)
+
+In a prod environment however, it will more likely be :
+
+    MONGOHQ_URL=adm1n:AStr0ngPassw0rd@10.0.4.4:27017
+
+
+## Declaring an entity ##
+
+Several conditions must be fulfilled to make a correct mapping between a golang struct and a mongo document :
+- All the mapped fields must be public
+- All the mapped fields must have the annotations `key` and `bson`
+- For a field, the value of the `bson` and `key` annotation must be equal
+- Among the fields, one and only one must have the annotated value '_id'
+
+Now, you've noticed that there are two different annotations, yet containing the exact same value, why is that ?
+Well, it's a choice a modeling : the value of these annotation represents the field of the document in the database, but the bson will use it to marshall and unmarshall documents into structs, whereas the key annotation will use it to build complex queries. In short : two purposes -> two annotations
+
+For example, given the entity :
+
+    type card struct {
+        Id bson.ObjectId `key:_id bson:_id` // The order doesn't matter, this field is the id of the object, must be unique
+        Height string    `key:height bson:height` // Height of the card
+        Color color      `key:color bson:color` // Color of the card
+    }
+
+The bson.ObjectId comes from mgo/bson, but you can very well have your own id :
+
+    type user struct {
+        Mail        string   `key:_id bson:_id`
+        Firstname   string   `key:fname bson:fname`
+        Lastname    string   `key:lname bson:lname`
+        Preferences []string `key:pref bson:pref`
+        age         int
+    }
+
+Here, the age field won't be persisted into the database, and all the users will have different mails.
+
+## Mongo Collection ##
+
+Please refer to the example to see the API in practice.

mgdb/manager.go

+package mgdb
+
+import (
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+	"os"
+)
+
+// errorConnection represents an error to be thrown upon connection
+type errorConnection struct {
+	s string
+}
+
+// Return the string contained in the error
+func (e *errorConnection) Error() string {
+	return e.s
+}
+
+// Creates a new error with the given message
+func newErrorConnection(message string) error {
+	return &errorConnection{message}
+}
+
+// MongoManager is aimed at handling the Mongo connection through the mgo driver
+type MongoManager struct {
+
+	// Session is the mgo.Session struct
+	Session *mgo.Session
+
+	// Database is the mgo.Database struct
+	Database *mgo.Database
+
+	Collections map[string]*MongoCollection
+}
+
+// NewManager a new Manager, the environment variable MONGOHQ_URL needs to be set
+// up with mongo uri, else it throws an error
+func NewManager(database string) (*MongoManager, error) {
+	uri := os.Getenv("MGDB_URL")
+	if uri == "" {
+		err := newErrorConnection("No uri provided, please set the MONGOHG_URL to connect to mongo")
+		return nil, err
+	}
+
+	sess, err := mgo.Dial(uri)
+
+	if err != nil {
+		return nil, err
+	}
+
+	db := sess.DB(database)
+	return &MongoManager{
+		sess,
+		db,
+		make(map[string]*MongoCollection),
+	}, nil
+}
+
+// Close closes the current connection
+// Be careful, you won't be able to query the Collections anymore
+func (m *MongoManager) Close() {
+	m.Session.Close()
+}
+
+// Get returns a MongoCollection over a specified Collection
+// The Collections are cached when they are called at least once
+func (m *MongoManager) Get(Collection string) *MongoCollection {
+	coll, ok := m.Collections[Collection]
+	if !ok {
+		coll = newCollection(m.Database.C(Collection))
+		m.Collections[Collection] = coll
+	}
+	return coll
+}
+
+// MongoCollection is a wrapped around an mgo Collection to query to database
+type MongoCollection struct {
+	// Collection is the mgo.Collection struct
+	Collection *mgo.Collection
+	factory    *MetadataFactory
+}
+
+// newCollection returns a new MongoCollection
+func newCollection(coll *mgo.Collection) *MongoCollection {
+	return &MongoCollection{
+		coll,
+		NewMetadataFactory(),
+	}
+}
+
+// Insert persists an Entity into the selected Collection
+// The _id field must be present in the mapping (see example provided)
+func (manager *MongoCollection) Insert(entity interface{}) (bool, error) {
+	err := manager.Collection.Insert(entity)
+	return err == nil, err
+}
+
+// UpdateByID updates the entity with the new value provided.
+// The _id of an Entity cannot be changed this way
+func (manager *MongoCollection) UpdateByID(entity interface{}) (bool, error) {
+	m := manager.factory.ToMap(entity)
+	err := manager.Collection.Update(map[string]interface{}{"_id": m["_id"]}, entity)
+	return err == nil, err
+}
+
+// UpdateAll updates the entities matching the selector with the query
+// The format of the parameters is expected to follow the one
+// provided in mgo's documentation
+// Return the number of updated entities
+func (manager *MongoCollection) UpdateAll(selector interface{}, update interface{}) (int, error) {
+	info, err := manager.Collection.UpdateAll(selector, update)
+	return info.Updated, err
+}
+
+// FindByID fill the entity from the document with matching id
+func (manager *MongoCollection) FindByID(id interface{}, result interface{}) error {
+	m := manager.factory.ToMap(id)
+	err := manager.Collection.Find(map[string]interface{}{"_id": m["_id"]}).One(result)
+	return err
+}
+
+// FindAll finds all entities matching the selector and put them into the result slice
+// The format of the selector is expected to follow the one
+// provided in mgo's documentation
+func (manager *MongoCollection) FindAll(query interface{}, result interface{}) error {
+	return manager.Collection.Find(query).All(result)
+}
+
+// DeleteByID deletes the entity matching the id
+// Return true if the delection was successful
+func (manager *MongoCollection) DeleteByID(id interface{}) (bool, error) {
+	m := manager.factory.ToMap(id)
+	err := manager.Collection.Remove(bson.M{"_id": m["_id"]})
+	return err == nil, err
+}
+
+// DeleteAll deletes all the entities matching the selector
+// The format of the selector is expected to follow the one
+// provided in mgo's documentation
+// Return the number of deleted entities
+func (manager *MongoCollection) DeleteAll(query interface{}) (int, error) {
+	info, err := manager.Collection.RemoveAll(query)
+	return info.Removed, err
+}
+
+// Count returns the number of entities currently in the Collection
+func (manager *MongoCollection) Count() int {
+	count, _ := manager.Collection.Count()
+	return count
+}
+
+// Drop drops the current Collection
+// This action is irreversible !
+func (manager *MongoCollection) Drop() error {
+	return manager.Collection.DropCollection()
+}

mgdb/manager_test.go

+package mgdb
+
+import (
+	"fmt"
+	"gopkg.in/mgo.v2/bson"
+	"os"
+	"testing"
+)
+
+type card struct {
+	ID    bson.ObjectId `key:"_id" bson:"_id"`
+	Value string        `key:"value" bson:"value"`
+	Color string        `key:"color" bson:"color"`
+}
+
+type hand struct {
+	ID      bson.ObjectId `key:"_id" bson:"_id"`
+	CardOne card          `key:"card_one" bson:"card_one"`
+	CardTwo card          `key:"card_two" bson:"card_two"`
+}
+
+var collection *MongoCollection
+var manager *MongoManager
+var err error
+
+func TestMain(m *testing.M) {
+	// Setup
+	fmt.Println("Try to connect to : " + os.Getenv("MGDB_URL"))
+
+	db := os.Getenv("DFSS_TEST")
+	if db == "" {
+		db = "demo"
+	}
+
+	manager, err = NewManager(db)
+
+	collection = manager.Get("demo")
+
+	// Run
+	code := m.Run()
+
+	// Teardown
+	// The collection is created automatically on
+	// first connection, that's why we do not recreate it manually
+	err = collection.Drop()
+
+	if err != nil {
+		fmt.Println("An error occurred while droping the collection")
+	}
+	manager.Close()
+
+	os.Exit(code)
+}
+
+func TestMongoConnection(t *testing.T) {
+	if err != nil {
+		t.Fatal("Couldn't connect to the database :", err)
+	}
+}
+
+func helperInsert(value, color string) (card, bool) {
+	c := card{
+		bson.NewObjectId(),
+		value,
+		color,
+	}
+
+	fmt.Println("Inserting card : ", c)
+	ok, err := collection.Insert(c)
+	if !ok {
+		fmt.Println("A problem occurred during insert : ", err)
+		return c, false
+	}
+	return c, true
+}
+
+func TestMongoInsert(t *testing.T) {
+	_, ok := helperInsert("five", "Hearts")
+	if !ok {
+		t.Fatal("Couldn't insert")
+	}
+}
+
+func TestMongoFindByID(t *testing.T) {
+	c, ok := helperInsert("king", "Spades")
+	if !ok {
+		t.Fatal("Couldn't insert")
+	}
+
+	res := card{}
+	err := collection.FindByID(c, &res)
+	if err != nil {
+		t.Fatal("Couldn't fetch the card : ", err)
+	}
+	fmt.Println("Fetched the card : ", res)
+}
+
+func TestMongoUpdateByID(t *testing.T) {
+	// Create and insert new card
+	c, ok := helperInsert("Ace", "Diamonds")
+	if !ok {
+		t.Fatal("Couldn't insert")
+	}
+
+	// Update and persist the card
+	c.Value = "Jack"
+	c.Color = ""
+	ok, err := collection.UpdateByID(c)
+	if !ok {
+		t.Fatal("Couldn't update the card : ", err)
+	}
+	fmt.Println("Updated to : ", c)
+
+	// Assert the changes have been persisted
+	res := card{}
+	err = collection.FindByID(c, &res)
+	if err != nil {
+		t.Fatal("Couldn't fetch the previously updated card")
+	}
+	fmt.Println("Fetched the card : ", res)
+
+	if c.ID != res.ID || c.Color != res.Color || c.Value != res.Value {
+		t.Fatal(fmt.Sprintf("Updated card with %v and fetched %v", c, res))
+	}
+}
+
+func TestMongoUpdateByIDNestedTypes(t *testing.T) {
+	// Create and insert a hand
+	c1 := card{bson.NewObjectId(), "Ace", "Spades"}
+	c2 := card{bson.NewObjectId(), "Ace", "Hearts"}
+	h := hand{bson.NewObjectId(), c1, c2}
+
+	ok, err := collection.Insert(h)
+	if !ok {
+		t.Fatal("Couldn't insert hand :", err)
+	}
+	fmt.Println("Hand is : ", h)
+
+	// Update the hand and persist the changes
+	h.CardOne.Value = "Three"
+	h.CardOne.Color = "Clubs"
+	h.CardTwo.Value = "Six"
+	h.CardTwo.Color = "Diamonds"
+	ok, err = collection.UpdateByID(h)
+	if !ok {
+		t.Fatal("An error occured while updating the hand :", err)
+	}
+	fmt.Println("Update hand to : ", h)
+
+	// Find the hand and assert the changes were made
+	res := hand{}
+	err = collection.FindByID(h, &res)
+	if err != nil {
+		t.Fatal("Couldn't fetch the previously update hand")
+	}
+	fmt.Println("Fetched hand : ", res)
+
+	if h.CardOne.Value != res.CardOne.Value || h.CardTwo.Value != res.CardTwo.Value || h.CardOne.Color != res.CardOne.Color || h.CardTwo.Color != res.CardTwo.Color {
+		t.Fatal(fmt.Sprintf("Fetched card is %v; expected %v", res, h))
+	}
+	fmt.Println("Update was successful")
+}
+
+func TestMongoDeleteByID(t *testing.T) {
+	c, ok := helperInsert("Three", "Hearts")
+	if !ok {
+		t.Fatal("Couldn't insert")
+	}
+
+	ok, err := collection.DeleteByID(c)
+	if !ok {
+		t.Fatal("Couldn't remove the card : ", err)
+	}
+	fmt.Println("Removed the card")
+}
+
+func ExampleMongoManager() {
+
+	//Define an animal to be use in further tests
+
+	type animal struct {
+		Name string `key:"_id" bson:"_id"`
+		Race string `key:"race" bson:"race"`
+		Age  int    `key:"age" bson:"age"`
+	}
+
+	//Initializes a MongoManager for the 'demo' database
+	manager, err := NewManager("demo")
+	if err != nil { /* Handle error */
+	}
+
+	// Connects to the collection named 'animals'
+	// If inexistant, it is created
+	animals := manager.Get("animals")
+
+	// Creates then insert a new animal into the collection
+	tails := animal{"Tails", "Fox", 15}
+	ok, _ := animals.Insert(tails)
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+
+	// Get the previously inserted animal
+	ani := animal{Name: "Tails"}
+	res := animal{}
+	err = animals.FindByID(ani, &res)
+	if err != nil { /* Handle error */
+	}
+
+	// res now contains the animal {"Tails", "Fox", 15}
+	// It is also possible to provided a struct with several field filled
+	// For example, the following code would have produced the same result
+	err = animals.FindByID(tails, &res)
+	if err != nil { /* Handle error */
+	}
+
+	// Update an entity and persist the changes in the database
+	res.Age += 2
+	ok, _ = animals.UpdateByID(res)
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+
+	// The database now contains the document {"_id": "Tails", "race": "Fox", age: 17}
+
+	ok, _ = animals.DeleteByID(res)
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+
+	// Tails has been successfully deleted from the database
+
+	// Insert a bunch of data for following examples
+
+	ok, _ = animals.Insert(animal{"Sonic", "Hedgehog", 12})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Eggman", "Robot", 15})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Amy", "Hedgehog", 12})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Tails", "Fox", 12})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Metal Sonic", "Robot", 14})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Knuckles", "Echidna", 13})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"EggRobo", "Robot", 15})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Tikal", "Echidna", 14})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Shadow", "Hedgehog", 13})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+	ok, _ = animals.Insert(animal{"Silver", "Hedgehog", 15})
+	fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
+
+	// Get all documents in the collection
+	var all []animal
+	err = animals.FindAll(bson.M{}, &all)
+	if err != nil { /* Handle error */
+	}
+
+	// Get all hedgehogs
+	// The type bson.M is provided by mgo/bson, it is an alias for map[string]interface{}
+	// To learn how to make a proper query, just refer to mongoDB documentation
+	var hedgehogs []animal
+	err = animals.FindAll(bson.M{"race": "Hedgehog"}, &hedgehogs)
+	if err != nil { /* Handle error */
+	}
+
+	// Fetch Tails, Eggman and Silver
+	var tailsEggmanSilver []animal
+	names := make([]string, 3)
+	names[0] = "Tails"
+	names[1] = "Eggman"
+	names[2] = "Silver"
+	err = animals.FindAll(bson.M{"_id": bson.M{"$in": names}}, &tailsEggmanSilver)
+	if err != nil { /* Handle error */
+	}
+
+	// Update all animals with age > 12 and decrement by one
+	// The first argument is used to select some documents, and the second argument contains the modification to apply
+	count, _ := animals.UpdateAll(bson.M{"age": bson.M{"$gt": 12}}, bson.M{"$inc": bson.M{"age": -1}})
+	fmt.Println(fmt.Sprintf("%d animals were uodated", count))
+
+	// UpdateAll animals with race = 'Robot' and change it to 'Machine'
+	count, _ = animals.UpdateAll(bson.M{"race": "Robot"}, bson.M{"$set": bson.M{"race": "Machine"}})
+	fmt.Println(fmt.Sprintf("%d animals were uodated", count))
+
+	// Delete all hedgehogs
+	count, _ = animals.DeleteAll(bson.M{"race": "Hedgehog"})
+	fmt.Println(fmt.Sprintf("%d animals were uodated", count))
+
+	// Drop all the collection
+	// Be careful when using this
+	err = animals.Drop()
+	if err != nil { /* Handle error */
+	}
+}

mgdb/mapping.go

+package mgdb
+
+import (
+	"reflect"
+)
+
+/****************
+ MetadataFactory
+****************/
+
+// MetadataFactory is a factory of metadata for structs
+// Metadata are stored in a map, indexed by the struct name
+type MetadataFactory struct {
+	metadatas map[string]*Metadata
+}
+
+// NewMetadataFactory instantiate a new empty factory
+func NewMetadataFactory() *MetadataFactory {
+	return &MetadataFactory{make(map[string]*Metadata)}
+}
+
+// Metadata get the Metadata associated to the struct
+// When querying with a yet unknown struct, the associated Metadata is built
+// If it is already known, just returns the stored Metadata
+func (factory *MetadataFactory) Metadata(element interface{}) *Metadata {
+	metadata, present := factory.metadatas[reflect.TypeOf(element).String()]
+	if !present {
+		metadata = newMetadata(element)
+		factory.metadatas[reflect.TypeOf(element).String()] = metadata
+	}
+	return metadata
+}
+
+// ToMap uses the metadata associated to the struct to returns the map
+// of the struct. Keys are the database fields and values are the values
+// stored in the struct
+func (factory *MetadataFactory) ToMap(element interface{}) map[string]interface{} {
+	data := factory.Metadata(element)
+	m := make(map[string]interface{})
+	v := reflect.ValueOf(element)
+	for key, value := range data.Mapping {
+		fieldValue := v.FieldByName(key).Interface()
+		m[value] = fieldValue
+	}
+	return m
+}
+
+/*********
+ Metadata
+*********/
+
+// Metadata represents the metadata for a struct
+type Metadata struct {
+
+	// Mapping maps the go fields to the database fields
+	Mapping map[string]string
+}
+
+// NewMetadata instantiate the Metadata associated to the given struct
+// It uses the `key` tag to do the mapping, more concrete
+// examples are provided in the documentation
+func newMetadata(element interface{}) *Metadata {
+	m := make(map[string]string)
+	t := reflect.TypeOf(element)
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+		if tag := field.Tag.Get("key"); tag != "" {
+			m[field.Name] = tag
+		}
+	}
+	return &Metadata{
+		m,
+	}
+}

mgdb/mapping_test.go

+package mgdb
+
+import (
+	"fmt"
+	"testing"
+)
+
+type animal struct {
+	Name string `key:"Name"`
+	Race string `key:"Race"`
+	Age  int    `key:"Age"`
+}
+
+type person struct {
+	firstName string `key:"f"`
+	lastName  string `key:"l"`
+	address   string `key:"a"`
+	ani       animal `key:"ani"`
+}
+
+type user struct {
+	mail     string `key:"_id"`
+	login    string `key:"login"`
+	password string
+}
+
+var factory = NewMetadataFactory()
+
+func TestPublicFields(t *testing.T) {
+	a := animal{"pika", "mouse", 15}
+	data := factory.Metadata(a)
+
+	if data == nil {
+		t.Fatal("Metadata is nil")
+	}
+
+	mapping := data.Mapping
+	fmt.Println(mapping)
+	if len(mapping) != 3 || mapping["Name"] != "Name" || mapping["Race"] != "Race" || mapping["Age"] != "Age" {
+		t.Fatal("Mapping not correctly built")
+	}
+}
+
+func TestPrivateFields(t *testing.T) {
+	p := person{"Ariel", "Undomiel", "Fondcombe", animal{}}
+	data := factory.Metadata(p)
+
+	if data == nil {
+		t.Fatal("Metadata is nil")
+	}
+
+	mapping := data.Mapping
+	fmt.Println(mapping)
+	if len(mapping) != 4 || mapping["firstName"] != "f" || mapping["lastName"] != "l" || mapping["address"] != "a" || mapping["ani"] != "ani" {
+		t.Fatal("Mapping not correctly built")
+	}
+}
+
+func TestUnmappedFields(t *testing.T) {
+	u := user{"sonic@hedgehog.io", "chaos", "emerald"}
+	data := factory.Metadata(u)
+
+	if data == nil {
+		t.Fatal("Metadata is nil")
+	}
+
+	mapping := data.Mapping
+	fmt.Println(mapping)
+	if len(mapping) != 2 || mapping["mail"] != "_id" || mapping["login"] != "login" {
+		t.Fatal("Mapping not correctly built")
+	}
+}