Commit 1410ae8f authored by Tristan Claverie's avatar Tristan Claverie
Browse files

Add library for mongoDB

parent 53fdbf52
Pipeline #93 passed with stage
......@@ -13,7 +13,11 @@ Unit tests:
# paths:
# - "coverage.html"
script:
- "go test -coverprofile c.out -v ./..."
- "go get gopkg.in/mgo.v2"
- "go test -coverprofile auth.part -v ./auth"
- "go test -coverprofile mgdb.part -v ./mgdb"
- "echo 'mode: set' *part > c.out"
- "grep -h -v 'mode: set' *part >> c.out"
- "go tool cover -html=c.out -o coverage.html"
ARM tests:
......@@ -21,7 +25,8 @@ ARM tests:
tags:
- arm
script:
- "go test -cover -short -v ./..."
- "go get gopkg.in/mgo.v2"
- "go test -cover -short -run 'Test[^M][^o][^n][^g][^o]' -v ./..."
Code lint:
stage: test
......@@ -31,5 +36,6 @@ Code lint:
- lint
script:
- "go get github.com/alecthomas/gometalinter"
- "go get gopkg.in/mgo.v2"
- "gometalinter --install"
- "gometalinter -t --deadline=60s ./..."
### DFSS - MGDB lib ###
This library is used in order to manage a connection to mongoDB
It uses the mgo driver, but aims at simplifying the queries to the database
## Mongo Manager ##
The struct handling the connection is MongoManager. It requires the environment variable MONGOHQ_URL in order to initialize a connection : it is the uri containing the informations to connect to mongo.
For example, in a test environment, we may have :
MONGOHQ_URL=localhost (require a mongo instance running on default port 27017)
In a prod environment however, it will more likely be :
MONGOHQ_URL=adm1n:AStr0ngPassw0rd@10.0.4.4:27017
## Declaring an entity ##
Several conditions must be fulfilled to make a correct mapping between a golang struct and a mongo document :
- All the mapped fields must be public
- All the mapped fields must have the annotations `key` and `bson`
- For a field, the value of the `bson` and `key` annotation must be equal
- Among the fields, one and only one must have the annotated value '_id'
Now, you've noticed that there are two different annotations, yet containing the exact same value, why is that ?
Well, it's a choice a modeling : the value of these annotation represents the field of the document in the database, but the bson will use it to marshall and unmarshall documents into structs, whereas the key annotation will use it to build complex queries. In short : two purposes -> two annotations
For example, given the entity :
type card struct {
Id bson.ObjectId `key:_id bson:_id` // The order doesn't matter, this field is the id of the object, must be unique
Height string `key:height bson:height` // Height of the card
Color color `key:color bson:color` // Color of the card
}
The bson.ObjectId comes from mgo/bson, but you can very well have your own id :
type user struct {
Mail string `key:_id bson:_id`
Firstname string `key:fname bson:fname`
Lastname string `key:lname bson:lname`
Preferences []string `key:pref bson:pref`
age int
}
Here, the age field won't be persisted into the database, and all the users will have different mails.
## Mongo Collection ##
Please refer to the example to see the API in practice.
package mgdb
import (
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
"os"
)
// errorConnection represents an error to be thrown upon connection
type errorConnection struct {
s string
}
// Return the string contained in the error
func (e *errorConnection) Error() string {
return e.s
}
// Creates a new error with the given message
func newErrorConnection(message string) error {
return &errorConnection{message}
}
// MongoManager is aimed at handling the Mongo connection through the mgo driver
type MongoManager struct {
// Session is the mgo.Session struct
Session *mgo.Session
// Database is the mgo.Database struct
Database *mgo.Database
Collections map[string]*MongoCollection
}
// NewManager a new Manager, the environment variable MONGOHQ_URL needs to be set
// up with mongo uri, else it throws an error
func NewManager(database string) (*MongoManager, error) {
uri := os.Getenv("MGDB_URL")
if uri == "" {
err := newErrorConnection("No uri provided, please set the MONGOHG_URL to connect to mongo")
return nil, err
}
sess, err := mgo.Dial(uri)
if err != nil {
return nil, err
}
db := sess.DB(database)
return &MongoManager{
sess,
db,
make(map[string]*MongoCollection),
}, nil
}
// Close closes the current connection
// Be careful, you won't be able to query the Collections anymore
func (m *MongoManager) Close() {
m.Session.Close()
}
// Get returns a MongoCollection over a specified Collection
// The Collections are cached when they are called at least once
func (m *MongoManager) Get(Collection string) *MongoCollection {
coll, ok := m.Collections[Collection]
if !ok {
coll = newCollection(m.Database.C(Collection))
m.Collections[Collection] = coll
}
return coll
}
// MongoCollection is a wrapped around an mgo Collection to query to database
type MongoCollection struct {
// Collection is the mgo.Collection struct
Collection *mgo.Collection
factory *MetadataFactory
}
// newCollection returns a new MongoCollection
func newCollection(coll *mgo.Collection) *MongoCollection {
return &MongoCollection{
coll,
NewMetadataFactory(),
}
}
// Insert persists an Entity into the selected Collection
// The _id field must be present in the mapping (see example provided)
func (manager *MongoCollection) Insert(entity interface{}) (bool, error) {
err := manager.Collection.Insert(entity)
return err == nil, err
}
// UpdateByID updates the entity with the new value provided.
// The _id of an Entity cannot be changed this way
func (manager *MongoCollection) UpdateByID(entity interface{}) (bool, error) {
m := manager.factory.ToMap(entity)
err := manager.Collection.Update(map[string]interface{}{"_id": m["_id"]}, entity)
return err == nil, err
}
// UpdateAll updates the entities matching the selector with the query
// The format of the parameters is expected to follow the one
// provided in mgo's documentation
// Return the number of updated entities
func (manager *MongoCollection) UpdateAll(selector interface{}, update interface{}) (int, error) {
info, err := manager.Collection.UpdateAll(selector, update)
return info.Updated, err
}
// FindByID fill the entity from the document with matching id
func (manager *MongoCollection) FindByID(id interface{}, result interface{}) error {
m := manager.factory.ToMap(id)
err := manager.Collection.Find(map[string]interface{}{"_id": m["_id"]}).One(result)
return err
}
// FindAll finds all entities matching the selector and put them into the result slice
// The format of the selector is expected to follow the one
// provided in mgo's documentation
func (manager *MongoCollection) FindAll(query interface{}, result interface{}) error {
return manager.Collection.Find(query).All(result)
}
// DeleteByID deletes the entity matching the id
// Return true if the delection was successful
func (manager *MongoCollection) DeleteByID(id interface{}) (bool, error) {
m := manager.factory.ToMap(id)
err := manager.Collection.Remove(bson.M{"_id": m["_id"]})
return err == nil, err
}
// DeleteAll deletes all the entities matching the selector
// The format of the selector is expected to follow the one
// provided in mgo's documentation
// Return the number of deleted entities
func (manager *MongoCollection) DeleteAll(query interface{}) (int, error) {
info, err := manager.Collection.RemoveAll(query)
return info.Removed, err
}
// Count returns the number of entities currently in the Collection
func (manager *MongoCollection) Count() int {
count, _ := manager.Collection.Count()
return count
}
// Drop drops the current Collection
// This action is irreversible !
func (manager *MongoCollection) Drop() error {
return manager.Collection.DropCollection()
}
package mgdb
import (
"fmt"
"gopkg.in/mgo.v2/bson"
"os"
"testing"
)
type card struct {
ID bson.ObjectId `key:"_id" bson:"_id"`
Value string `key:"value" bson:"value"`
Color string `key:"color" bson:"color"`
}
type hand struct {
ID bson.ObjectId `key:"_id" bson:"_id"`
CardOne card `key:"card_one" bson:"card_one"`
CardTwo card `key:"card_two" bson:"card_two"`
}
var collection *MongoCollection
var manager *MongoManager
var err error
func TestMain(m *testing.M) {
// Setup
fmt.Println("Try to connect to : " + os.Getenv("MGDB_URL"))
db := os.Getenv("DFSS_TEST")
if db == "" {
db = "demo"
}
manager, err = NewManager(db)
collection = manager.Get("demo")
// Run
code := m.Run()
// Teardown
// The collection is created automatically on
// first connection, that's why we do not recreate it manually
err = collection.Drop()
if err != nil {
fmt.Println("An error occurred while droping the collection")
}
manager.Close()
os.Exit(code)
}
func TestMongoConnection(t *testing.T) {
if err != nil {
t.Fatal("Couldn't connect to the database :", err)
}
}
func helperInsert(value, color string) (card, bool) {
c := card{
bson.NewObjectId(),
value,
color,
}
fmt.Println("Inserting card : ", c)
ok, err := collection.Insert(c)
if !ok {
fmt.Println("A problem occurred during insert : ", err)
return c, false
}
return c, true
}
func TestMongoInsert(t *testing.T) {
_, ok := helperInsert("five", "Hearts")
if !ok {
t.Fatal("Couldn't insert")
}
}
func TestMongoFindByID(t *testing.T) {
c, ok := helperInsert("king", "Spades")
if !ok {
t.Fatal("Couldn't insert")
}
res := card{}
err := collection.FindByID(c, &res)
if err != nil {
t.Fatal("Couldn't fetch the card : ", err)
}
fmt.Println("Fetched the card : ", res)
}
func TestMongoUpdateByID(t *testing.T) {
// Create and insert new card
c, ok := helperInsert("Ace", "Diamonds")
if !ok {
t.Fatal("Couldn't insert")
}
// Update and persist the card
c.Value = "Jack"
c.Color = ""
ok, err := collection.UpdateByID(c)
if !ok {
t.Fatal("Couldn't update the card : ", err)
}
fmt.Println("Updated to : ", c)
// Assert the changes have been persisted
res := card{}
err = collection.FindByID(c, &res)
if err != nil {
t.Fatal("Couldn't fetch the previously updated card")
}
fmt.Println("Fetched the card : ", res)
if c.ID != res.ID || c.Color != res.Color || c.Value != res.Value {
t.Fatal(fmt.Sprintf("Updated card with %v and fetched %v", c, res))
}
}
func TestMongoUpdateByIDNestedTypes(t *testing.T) {
// Create and insert a hand
c1 := card{bson.NewObjectId(), "Ace", "Spades"}
c2 := card{bson.NewObjectId(), "Ace", "Hearts"}
h := hand{bson.NewObjectId(), c1, c2}
ok, err := collection.Insert(h)
if !ok {
t.Fatal("Couldn't insert hand :", err)
}
fmt.Println("Hand is : ", h)
// Update the hand and persist the changes
h.CardOne.Value = "Three"
h.CardOne.Color = "Clubs"
h.CardTwo.Value = "Six"
h.CardTwo.Color = "Diamonds"
ok, err = collection.UpdateByID(h)
if !ok {
t.Fatal("An error occured while updating the hand :", err)
}
fmt.Println("Update hand to : ", h)
// Find the hand and assert the changes were made
res := hand{}
err = collection.FindByID(h, &res)
if err != nil {
t.Fatal("Couldn't fetch the previously update hand")
}
fmt.Println("Fetched hand : ", res)
if h.CardOne.Value != res.CardOne.Value || h.CardTwo.Value != res.CardTwo.Value || h.CardOne.Color != res.CardOne.Color || h.CardTwo.Color != res.CardTwo.Color {
t.Fatal(fmt.Sprintf("Fetched card is %v; expected %v", res, h))
}
fmt.Println("Update was successful")
}
func TestMongoDeleteByID(t *testing.T) {
c, ok := helperInsert("Three", "Hearts")
if !ok {
t.Fatal("Couldn't insert")
}
ok, err := collection.DeleteByID(c)
if !ok {
t.Fatal("Couldn't remove the card : ", err)
}
fmt.Println("Removed the card")
}
func ExampleMongoManager() {
//Define an animal to be use in further tests
type animal struct {
Name string `key:"_id" bson:"_id"`
Race string `key:"race" bson:"race"`
Age int `key:"age" bson:"age"`
}
//Initializes a MongoManager for the 'demo' database
manager, err := NewManager("demo")
if err != nil { /* Handle error */
}
// Connects to the collection named 'animals'
// If inexistant, it is created
animals := manager.Get("animals")
// Creates then insert a new animal into the collection
tails := animal{"Tails", "Fox", 15}
ok, _ := animals.Insert(tails)
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
// Get the previously inserted animal
ani := animal{Name: "Tails"}
res := animal{}
err = animals.FindByID(ani, &res)
if err != nil { /* Handle error */
}
// res now contains the animal {"Tails", "Fox", 15}
// It is also possible to provided a struct with several field filled
// For example, the following code would have produced the same result
err = animals.FindByID(tails, &res)
if err != nil { /* Handle error */
}
// Update an entity and persist the changes in the database
res.Age += 2
ok, _ = animals.UpdateByID(res)
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
// The database now contains the document {"_id": "Tails", "race": "Fox", age: 17}
ok, _ = animals.DeleteByID(res)
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
// Tails has been successfully deleted from the database
// Insert a bunch of data for following examples
ok, _ = animals.Insert(animal{"Sonic", "Hedgehog", 12})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Eggman", "Robot", 15})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Amy", "Hedgehog", 12})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Tails", "Fox", 12})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Metal Sonic", "Robot", 14})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Knuckles", "Echidna", 13})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"EggRobo", "Robot", 15})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Tikal", "Echidna", 14})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Shadow", "Hedgehog", 13})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
ok, _ = animals.Insert(animal{"Silver", "Hedgehog", 15})
fmt.Println(fmt.Sprintf("Transaction went ok : %v", ok))
// Get all documents in the collection
var all []animal
err = animals.FindAll(bson.M{}, &all)
if err != nil { /* Handle error */
}
// Get all hedgehogs
// The type bson.M is provided by mgo/bson, it is an alias for map[string]interface{}
// To learn how to make a proper query, just refer to mongoDB documentation
var hedgehogs []animal
err = animals.FindAll(bson.M{"race": "Hedgehog"}, &hedgehogs)
if err != nil { /* Handle error */
}
// Fetch Tails, Eggman and Silver
var tailsEggmanSilver []animal
names := make([]string, 3)
names[0] = "Tails"
names[1] = "Eggman"
names[2] = "Silver"
err = animals.FindAll(bson.M{"_id": bson.M{"$in": names}}, &tailsEggmanSilver)
if err != nil { /* Handle error */
}
// Update all animals with age > 12 and decrement by one
// The first argument is used to select some documents, and the second argument contains the modification to apply
count, _ := animals.UpdateAll(bson.M{"age": bson.M{"$gt": 12}}, bson.M{"$inc": bson.M{"age": -1}})
fmt.Println(fmt.Sprintf("%d animals were uodated", count))
// UpdateAll animals with race = 'Robot' and change it to 'Machine'
count, _ = animals.UpdateAll(bson.M{"race": "Robot"}, bson.M{"$set": bson.M{"race": "Machine"}})
fmt.Println(fmt.Sprintf("%d animals were uodated", count))
// Delete all hedgehogs
count, _ = animals.DeleteAll(bson.M{"race": "Hedgehog"})
fmt.Println(fmt.Sprintf("%d animals were uodated", count))
// Drop all the collection
// Be careful when using this
err = animals.Drop()
if err != nil { /* Handle error */
}
}
package mgdb
import (
"reflect"
)
/****************
MetadataFactory
****************/
// MetadataFactory is a factory of metadata for structs
// Metadata are stored in a map, indexed by the struct name
type MetadataFactory struct {
metadatas map[string]*Metadata
}
// NewMetadataFactory instantiate a new empty factory
func NewMetadataFactory() *MetadataFactory {
return &MetadataFactory{make(map[string]*Metadata)}
}
// Metadata get the Metadata associated to the struct
// When querying with a yet unknown struct, the associated Metadata is built
// If it is already known, just returns the stored Metadata
func (factory *MetadataFactory) Metadata(element interface{}) *Metadata {
metadata, present := factory.metadatas[reflect.TypeOf(element).String()]
if !present {
metadata = newMetadata(element)
factory.metadatas[reflect.TypeOf(element).String()] = metadata
}
return metadata
}
// ToMap uses the metadata associated to the struct to returns the map
// of the struct. Keys are the database fields and values are the values
// stored in the struct
func (factory *MetadataFactory) ToMap(element interface{}) map[string]interface{} {
data := factory.Metadata(element)
m := make(map[string]interface{})
v := reflect.ValueOf(element)
for key, value := range data.Mapping {
fieldValue := v.FieldByName(key).Interface()
m[value] = fieldValue
}
return m
}
/*********
Metadata
*********/
// Metadata represents the metadata for a struct
type Metadata struct {
// Mapping maps the go fields to the database fields
Mapping map[string]string
}
// NewMetadata instantiate the Metadata associated to the given struct
// It uses the `key` tag to do the mapping, more concrete
// examples are provided in the documentation
func newMetadata(element interface{}) *Metadata {
m := make(map[string]string)
t := reflect.TypeOf(element)
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
if tag := field.Tag.Get("key"); tag != "" {
m[field.Name] = tag
}
}
return &Metadata{
m,
}
}
package mgdb
import (
"fmt"
"testing"
)
type animal struct {
Name string `key:"Name"`
Race string `key:"Race"`
Age int `key:"Age"`
}
type person struct {
firstName string `key:"f"`