Fix LDAP sync. We are ready for merge into master

compose/karmen/docker-compose.yml

@@ -23,7 +23,7 @@ services:
   ldap_server:
     restart: always
     build: ./ldap
-    command: --loglevel debug
+    # command: --loglevel debug
     domainname: foo.org
     hostname: foo.org
     ports:

compose/karmen/ldap/bootstrap/ldif/foo.org.ldif

@@ -6,6 +6,8 @@ objectclass: organizationalUnit
 objectclass: top
 ou: Groups
 
+## Mailing lists
+
 dn: ou=Mailing Lists,ou=Groups,dc=foo,dc=org
 changetype: add
 objectclass: organizationalUnit
@@ -19,20 +21,58 @@ member: cn=John DOE,ou=People,dc=foo,dc=org
 objectclass: groupOfNames
 objectclass: top
 
-dn: cn=Administratives,ou=Groups,dc=foo,dc=org
+dn: cn=LINGUE STRANIERE,ou=Mailing Lists,ou=Groups,dc=foo,dc=org
 changetype: add
-cn: Administratives
+cn: LINGUE STRANIERE
 member: cn=John DOE,ou=People,dc=foo,dc=org
 objectclass: groupOfNames
 objectclass: top
 
-dn: cn=Teachers,ou=Groups,dc=foo,dc=org
+dn: cn=SOSTEGNO,ou=Mailing Lists,ou=Groups,dc=foo,dc=org
 changetype: add
-cn: Teachers
+cn: SOSTEGNO
 member: cn=John DOE,ou=People,dc=foo,dc=org
 objectclass: groupOfNames
 objectclass: top
 
+## Groups
+
+dn: cn=Teachers,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: Teachers
+gidNumber: 6000
+objectclass: posixGroup
+objectclass: top
+
+dn: cn=Administratives,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: Administratives
+gidNumber: 6001
+objectclass: posixGroup
+objectclass: top
+
+dn: ou=Dipartimenti,ou=Groups,dc=foo,dc=org
+changetype: add
+ou: Dipartimenti
+objectclass: organizationalUnit
+objectclass: top
+
+dn: cn=LINGUE STRANIERE,ou=Dipartimenti,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: LINGUE STRANIERE
+gidNumber: 6003
+objectclass: posixGroup
+objectclass: top
+
+dn: cn=SOSTEGNO,ou=Dipartimenti,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: SOSTEGNO
+gidNumber: 6004
+objectclass: posixGroup
+objectclass: top
+
+## People
+
 dn: ou=People,dc=foo,dc=org
 changetype: add
 objectclass: organizationalUnit

config/config.go

@@ -83,6 +83,11 @@ type ConfigT struct {
 
 		TeachersSearchBase        string `yaml:"teachers_search_base"`
 		AdministrativesSearchBase string `yaml:"administratives_search_base"`
+
+		TeachersGroup        string `yaml:"teachers_group"`
+		AdministrativesGroup string `yaml:"administratives_group"`
+
+		TeachersML string `yaml:"teachers_ml"`
 	}
 }
 

cron/sync/sync.go

@@ -66,13 +66,41 @@ func (slice SliceToStringSlicer) Convert() (result []string) {
 }
 
 type Result struct {
+	Description string
+
 	Added   []orm.User
 	Updated []orm.User
 	Removed []string
 }
 
 func (result *Result) String() string {
-	return fmt.Sprintf("[sync] %d added, %d updated, %d removed", result.Added, result.Updated, result.Removed)
+	output := fmt.Sprintf("\n%s\n", result.Description)
+	if len(result.Added) > 0 {
+		output += "\n== ADDED ==\n\n"
+		for _, user := range result.Added {
+			output += fmt.Sprintf("* %s\n", user.CompleteName())
+		}
+	}
+	if len(result.Updated) > 0 {
+		output += "\n== UPDATED ==\n\n"
+		for _, user := range result.Updated {
+			output += fmt.Sprintf("* %s\n", user.CompleteName())
+		}
+	}
+	if len(result.Removed) > 0 {
+		output += "\n== REMOVED ==\n\n"
+		for _, user := range result.Removed {
+			output += fmt.Sprintf("* %s\n", user)
+		}
+	}
+
+	if len(result.Added)*len(result.Updated)*len(result.Removed) > 0 {
+		output += "\n"
+	}
+
+	output += fmt.Sprintf("[%d added, %d updated, %d removed]", len(result.Added), len(result.Updated), len(result.Removed))
+
+	return output
 }
 
 type SyncJob struct {
@@ -127,6 +155,8 @@ func (syncJob *SyncJob) SyncUsers(ldapClient *karmen_ldap.Client, karmenClient *
 
 	result := new(Result)
 
+	result.Description = "Sync users..."
+
 	for _, a := range actions {
 		switch a.Type {
 		case slicediff.Remove:
@@ -144,6 +174,9 @@ func (syncJob *SyncJob) SyncUsers(ldapClient *karmen_ldap.Client, karmenClient *
 			if !user.GetExclude() && user.GetRegenerate() {
 				result.Updated = append(result.Updated, user)
 			}
+			if user.GetExclude() {
+				result.Removed = append(result.Removed, user.DN())
+			}
 
 		}
 	}
@@ -240,26 +273,34 @@ func (syncJob *SyncJob) SyncUsers(ldapClient *karmen_ldap.Client, karmenClient *
 }
 
 func (syncJob *SyncJob) SyncGroup(ldapClient *karmen_ldap.Client, users []orm.User, groupDN string) (*Result, error) {
+	var ml bool
+
+	result := new(Result)
+
+	result.Description = fmt.Sprintf("Sync group %s", groupDN)
+
 	actions := make(map[string]*slicediff.Action)
 	entries, err := ldapClient.GroupMembers(groupDN)
 	if err != nil {
 		return nil, err
 	}
 
-	values := entries[0].Attributes[0].Values
+	values := make([]string, 0)
+
+	if len(entries) > 0 {
+		values = entries[0].Attributes[0].Values
+	}
 
 	if strings.Contains(groupDN, "Mailing Lists") {
+		ml = true
 		actions = slicediff.Diff(SliceToStringSlicer(values).Convert, usersToDNConverter(users).Convert)
 	} else {
 		actions = slicediff.Diff(SliceToStringSlicer(values).Convert, usersToUsernamesConverter(users).Convert)
 	}
 
-	result := new(Result)
-
 	for _, a := range actions {
 		switch a.Type {
 		case slicediff.Remove:
-			// entry := entries[a.Id]
 			result.Removed = append(result.Removed, values[a.Id])
 
 		case slicediff.Add:
@@ -267,6 +308,11 @@ func (syncJob *SyncJob) SyncGroup(ldapClient *karmen_ldap.Client, users []orm.Us
 			if !user.GetExclude() {
 				result.Added = append(result.Added, user)
 			}
+		case slicediff.Update:
+			user := users[a.Id]
+			if user.GetExclude() && !ml {
+				result.Removed = append(result.Removed, user.Username())
+			}
 
 		}
 	}
@@ -316,10 +362,10 @@ func (syncJob *SyncJob) Run() {
 	}
 
 	log.Println("Retrieving Departments...")
-	// // departments, err := karmenClient.GetDepartments()
-	// // if err != nil {
-	// // 	log.Println(err)
-	// // }
+	departments, err := karmenClient.GetDepartments()
+	if err != nil {
+		log.Println(err)
+	}
 
 	users := make([]orm.User, 0)
 	for _, teacher := range teachers {
@@ -331,39 +377,36 @@ func (syncJob *SyncJob) Run() {
 	} else {
 		log.Println(result)
 	}
-	// log.Println("Sync 'Docenti' group...")
-	// if result, err := syncJob.SyncGroup(ldapClient, users, "cn=Docenti"); err != nil {
-	// 	panic(err)
-	// } else {
-	// 	log.Println(result)
-	// }
-
-	// log.Println("Sync 'Tutti i docenti' ML...")
-	// if result, err := syncJob.SyncGroup(ldapClient, users, "cn=Tutti i docenti,ou=Mailing Lists"); err != nil {
-	// 	panic(err)
-	// } else {
-	// 	log.Println(result)
-	// }
-
-	// log.Println("Sync Departments and MLs...")
-	// for _, department := range departments {
-	// 	users := make([]orm.User, 0)
-	// 	for _, teacher := range department.Teachers {
-	// 		users = append(users, teacher)
-	// 	}
-	// 	group := fmt.Sprintf("cn=%s,ou=Mailing Lists", department.Name)
-	// 	if result, err := syncJob.SyncGroup(ldapClient, users, group); err != nil {
-	// 		panic(err)
-	// 	} else {
-	// 		log.Println(result)
-	// 	}
-	// 	group = fmt.Sprintf("cn=%s,ou=Dipartimenti", department.Name)
-	// 	if result, err := syncJob.SyncGroup(ldapClient, users, group); err != nil {
-	// 		panic(err)
-	// 	} else {
-	// 		log.Println(result)
-	// 	}
-	// }
+	if result, err := syncJob.SyncGroup(ldapClient, users, syncJob.conf.Sync.TeachersGroup); err != nil {
+		panic(err)
+	} else {
+		log.Println(result)
+	}
+
+	if result, err := syncJob.SyncGroup(ldapClient, users, syncJob.conf.Sync.TeachersML); err != nil {
+		panic(err)
+	} else {
+		log.Println(result)
+	}
+
+	for _, department := range departments {
+		users := make([]orm.User, 0)
+		for _, teacher := range department.Teachers {
+			users = append(users, teacher)
+		}
+		group := fmt.Sprintf("cn=%s,ou=Mailing Lists", department.Name)
+		if result, err := syncJob.SyncGroup(ldapClient, users, group); err != nil {
+			panic(err)
+		} else {
+			log.Println(result)
+		}
+		group = fmt.Sprintf("cn=%s,ou=Dipartimenti", department.Name)
+		if result, err := syncJob.SyncGroup(ldapClient, users, group); err != nil {
+			panic(err)
+		} else {
+			log.Println(result)
+		}
+	}
 
 	administratives, err := karmenClient.GetAdministratives()
 	if err != nil {
@@ -387,4 +430,10 @@ func (syncJob *SyncJob) Run() {
 		log.Println(result)
 	}
 
+	if result, err := syncJob.SyncGroup(ldapClient, users, syncJob.conf.Sync.AdministrativesGroup); err != nil {
+		panic(err)
+	} else {
+		log.Println(result)
+	}
+
 }

cron/sync/test/config.yaml

@@ -20,4 +20,8 @@ ldap:
 sync:
   safe_run: false
   send_mail: false
-  verbose: false
+  teachers_search_base: "ou=Teachers,ou=People,dc=foo,dc=org"
+  administratives_search_base: "ou=Administratives,ou=People,dc=foo,dc=org"
+  teachers_group: "cn=Teachers"
+  administratives_group: "cn=Administratives"
+  teachers_ml: "cn=Teachers,ou=Mailing Lists"

cron/sync/test/karmen_test_sync/image/bootstrap/ldif/foo.org.ldif

@@ -6,6 +6,8 @@ objectclass: organizationalUnit
 objectclass: top
 ou: Groups
 
+## Mailing lists
+
 dn: ou=Mailing Lists,ou=Groups,dc=foo,dc=org
 changetype: add
 objectclass: organizationalUnit
@@ -19,20 +21,58 @@ member: cn=John DOE,ou=People,dc=foo,dc=org
 objectclass: groupOfNames
 objectclass: top
 
-dn: cn=Administratives,ou=Groups,dc=foo,dc=org
+dn: cn=LINGUE STRANIERE,ou=Mailing Lists,ou=Groups,dc=foo,dc=org
 changetype: add
-cn: Administratives
+cn: LINGUE STRANIERE
 member: cn=John DOE,ou=People,dc=foo,dc=org
 objectclass: groupOfNames
 objectclass: top
 
-dn: cn=Teachers,ou=Groups,dc=foo,dc=org
+dn: cn=SOSTEGNO,ou=Mailing Lists,ou=Groups,dc=foo,dc=org
 changetype: add
-cn: Teachers
+cn: SOSTEGNO
 member: cn=John DOE,ou=People,dc=foo,dc=org
 objectclass: groupOfNames
 objectclass: top
 
+## Groups
+
+dn: cn=Teachers,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: Teachers
+gidNumber: 6000
+objectclass: posixGroup
+objectclass: top
+
+dn: cn=Administratives,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: Administratives
+gidNumber: 6001
+objectclass: posixGroup
+objectclass: top
+
+dn: ou=Dipartimenti,ou=Groups,dc=foo,dc=org
+changetype: add
+ou: Dipartimenti
+objectclass: organizationalUnit
+objectclass: top
+
+dn: cn=LINGUE STRANIERE,ou=Dipartimenti,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: LINGUE STRANIERE
+gidNumber: 6003
+objectclass: posixGroup
+objectclass: top
+
+dn: cn=SOSTEGNO,ou=Dipartimenti,ou=Groups,dc=foo,dc=org
+changetype: add
+cn: SOSTEGNO
+gidNumber: 6004
+objectclass: posixGroup
+objectclass: top
+
+## People
+
 dn: ou=People,dc=foo,dc=org
 changetype: add
 objectclass: organizationalUnit

cron/sync/test/sync_test.go

@@ -1,6 +1,7 @@
 package sync_test
 
 import (
+	"fmt"
 	"log"
 	"testing"
 	"time"
@@ -64,7 +65,7 @@ func (t *testSuite) TestSyncUsers() {
 	teachers, err := karmen.GetTeachers()
 	t.Nil(err)
 
-	entries, err := ldap.Users("ou=Teachers,ou=People,dc=foo,dc=org")
+	entries, err := ldap.Users(conf.Sync.TeachersSearchBase)
 	t.Nil(err)
 
 	users := make([]orm.User, 0)
@@ -88,7 +89,7 @@ func (t *testSuite) TestSyncUsers() {
 	teachers, err = karmen.GetTeachers()
 	t.Nil(err)
 
-	entries, err = ldap.Users("ou=Teachers,ou=People,dc=foo,dc=org")
+	entries, err = ldap.Users(conf.Sync.TeachersSearchBase)
 	t.Nil(err)
 
 	users = make([]orm.User, 0)
@@ -117,7 +118,7 @@ func (t *testSuite) TestSyncUsers() {
 	teachers, err = karmen.GetTeachers()
 	t.Nil(err)
 
-	entries, err = ldap.Users("ou=Teachers,ou=People,dc=foo,dc=org")
+	entries, err = ldap.Users(conf.Sync.TeachersSearchBase)
 	t.Nil(err)
 
 	users = make([]orm.User, 0)
@@ -138,7 +139,7 @@ func (t *testSuite) TestSyncUsers() {
 	administratives, err := karmen.GetAdministratives()
 	t.Nil(err)
 
-	entries, err = ldap.Users("ou=Administratives,ou=People,dc=foo,dc=org")
+	entries, err = ldap.Users(conf.Sync.AdministrativesSearchBase)
 	t.Nil(err)
 
 	users = make([]orm.User, 0)
@@ -168,7 +169,29 @@ func (t *testSuite) TestSyncGroup() {
 		users = append(users, teacher)
 	}
 
-	result, err := syncJob.SyncGroup(ldap, users, "cn=Teachers")
+	result, err := syncJob.SyncGroup(ldap, users, conf.Sync.TeachersGroup)
+	t.Nil(err)
+
+	if !t.Failed() {
+		t.Equal(11, len(result.Added))
+		t.Equal(0, len(result.Updated))
+		t.Equal(0, len(result.Removed))
+	}
+
+	// Running twice should result in no update
+
+	result, err = syncJob.SyncGroup(ldap, users, conf.Sync.TeachersGroup)
+	t.Nil(err)
+
+	if !t.Failed() {
+		t.Equal(0, len(result.Added))
+		t.Equal(0, len(result.Updated))
+		t.Equal(0, len(result.Removed))
+	}
+
+	// Test ML sync
+
+	result, err = syncJob.SyncGroup(ldap, users, conf.Sync.TeachersML)
 	t.Nil(err)
 
 	if !t.Failed() {
@@ -176,4 +199,74 @@ func (t *testSuite) TestSyncGroup() {
 		t.Equal(0, len(result.Updated))
 		t.Equal(1, len(result.Removed))
 	}
+
+	// Running twice should result in no update
+
+	result, err = syncJob.SyncGroup(ldap, users, conf.Sync.TeachersML)
+	t.Nil(err)
+
+	if !t.Failed() {
+		t.Equal(0, len(result.Added))
+		t.Equal(0, len(result.Updated))
+		t.Equal(0, len(result.Removed))
+	}
+
+	administratives, err := karmen.GetAdministratives()
+	t.Nil(err)
+
+	users = make([]orm.User, 0)
+	for _, administrative := range administratives {
+		users = append(users, administrative)
+	}
+
+	result, err = syncJob.SyncGroup(ldap, users, conf.Sync.AdministrativesGroup)
+	t.Nil(err)
+
+	if !t.Failed() {
+		t.Equal(2, len(result.Added))
+		t.Equal(0, len(result.Updated))
+		t.Equal(0, len(result.Removed))
+	}
+
+	departments, err := karmen.GetDepartments()
+	t.Nil(err)
+
+	if !t.Failed() {
+
+		results := make(map[string][]int)
+
+		results["cn=SOSTEGNO,ou=Mailing Lists"] = []int{2, 0, 1}
+		results["cn=SOSTEGNO,ou=Dipartimenti"] = []int{2, 0, 0}
+
+		results["cn=LINGUE STRANIERE,ou=Mailing Lists"] = []int{1, 0, 1}
+		results["cn=LINGUE STRANIERE,ou=Dipartimenti"] = []int{1, 0, 0}
+
+		for _, department := range departments {
+			users := make([]orm.User, 0)
+			for _, teacher := range department.Teachers {
+				users = append(users, teacher)
+			}
+			group := fmt.Sprintf("cn=%s,ou=Mailing Lists", department.Name)
+			result, err := syncJob.SyncGroup(ldap, users, group)
+			t.Nil(err)
+
+			if !t.Failed() {
+				t.Equal(results[group][0], len(result.Added))
+				t.Equal(results[group][1], len(result.Updated))
+				t.Equal(results[group][2], len(result.Removed))
+			}
+
+			group = fmt.Sprintf("cn=%s,ou=Dipartimenti", department.Name)
+			result, err = syncJob.SyncGroup(ldap, users, group)
+			t.Nil(err)
+
+			if !t.Failed() {
+				t.Equal(results[group][0], len(result.Added))
+				t.Equal(results[group][1], len(result.Updated))
+				t.Equal(results[group][2], len(result.Removed))
+			}
+
+		}
+	}
+
 }