feat: impl ImportCourses for BrightspaceService (#479)

* feat: add initial csvs local directory and default brightspace image for integration of courses

* fix: modify gorm criteria for ProviderPlatform state to be enabled

* feat: add Brightspace ProviderPlatformType for integration of courses

* feat: add gocarina/gocvs library used for reading csv files for integration of courses

* feat: add logic for the integration of brightspace courses

* fix: use brightspace default img path

---------

Co-authored-by: PThorpe92 <preston@unlockedlabs.org>

backend/src/database/provider_platforms.go

@@ -36,7 +36,7 @@ func iterMap[T any](fun func(T) T, arr []T) []T {
 func (db *DB) GetAllActiveProviderPlatforms() ([]models.ProviderPlatform, error) {
 	var platforms []models.ProviderPlatform
 	if err := db.Model(models.ProviderPlatform{}).Preload("OidcClient").
-		Find(&platforms, "state = ?", "active").Error; err != nil {
+		Find(&platforms, "state = ?", "enabled").Error; err != nil {
 		return nil, newGetRecordsDBError(err, "provider_platforms")
 	}
 

backend/src/models/provider_platforms.go

@@ -21,6 +21,7 @@ const (
 	CanvasOSS   ProviderPlatformType = "canvas_oss"
 	CanvasCloud ProviderPlatformType = "canvas_cloud"
 	Kolibri     ProviderPlatformType = "kolibri"
+	Brightspace ProviderPlatformType = "brightspace"
 )
 
 type ProviderPlatformState string

go.work.sum

@@ -18,6 +18,8 @@ github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6v
 github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=

provider-middleware/brightspace.go

@@ -2,17 +2,21 @@ package main
 
 import (
 	"UnlockEdv2/src/models"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/url"
 	"os"
 	"strings"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"gorm.io/gorm"
 )
 
 const (
+	CsvDownloadPath     = "csvs"
 	TokenEndpoint       = "https://auth.brightspace.com/core/connect/token"
 	DataSetsEndpoint    = "https://unlocked.brightspacedemo.com/d2l/api/lp/1.28/dataExport/bds/list"
 	DataDownloadEnpoint = "https://unlocked.brightspacedemo.com/d2l/api/lp/1.28/dataExport/bds/download/%s"
@@ -31,8 +35,7 @@ type BrightspaceService struct {
 	JobParams          *map[string]interface{}
 }
 
-// for linting reasons changed new to New below temporarily so this can be reviewed,
-func NewBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, params *map[string]interface{}) (*BrightspaceService, error) {
+func newBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, params *map[string]interface{}) (*BrightspaceService, error) {
 	keysSplit := strings.Split(provider.AccessKey, ";")
 	if len(keysSplit) < 2 {
 		return nil, errors.New("unable to find refresh token, unable to intialize BrightspaceService")
@@ -42,23 +45,50 @@ func NewBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, param
 		return nil, errors.New("no brightspace scope found, unable to intialize BrightspaceService")
 	}
 	brightspaceService := BrightspaceService{
-		//brightspace - set fields
-		JobParams: params,
+		ProviderPlatformID: provider.ID,
+		Client:             &http.Client{},
+		BaseURL:            provider.BaseUrl,
+		ClientID:           provider.AccountID,
+		ClientSecret:       keysSplit[0],
+		RefreshToken:       keysSplit[1],
+		Scope:              scope,
+		JobParams:          params,
 	}
 	data := url.Values{}
 	data.Add("grant_type", "refresh_token")
 	data.Add("refresh_token", brightspaceService.RefreshToken)
 	data.Add("client_id", brightspaceService.ClientID)
 	data.Add("client_secret", brightspaceService.ClientSecret)
 	data.Add("scope", brightspaceService.Scope)
-	//a send post request to brightspace
-	//b parse response for access_token and refresh_token
-	//c save new refresh_token (tack it onto the end of the client secret separated by semicolon)
+	log.Infof("refreshing token using endpoint url %v", TokenEndpoint)
+	resp, err := brightspaceService.SendPostRequest(TokenEndpoint, data)
+	if err != nil {
+		log.Errorf("error sending post request to url %v", TokenEndpoint)
+		return nil, err
+	}
+	var tokenMap map[string]interface{}
+	if err := json.NewDecoder(resp.Body).Decode(&tokenMap); err != nil {
+		log.Errorf("error decoding to response from url %v, error is: %v", TokenEndpoint, err)
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		errType, okError := tokenMap["error"].(string)
+		errMsg, okDesc := tokenMap["error_description"].(string)
+		msg := "unable to request new refresh token from brightspace"
+		if okError && okDesc {
+			msg = fmt.Sprintf("unable to request new refresh token from brightspace, response error message is: %s: %s", errType, errMsg)
+			return nil, errors.New(msg)
+		}
+		return nil, errors.New(msg)
+	}
+	brightspaceService.AccessToken = tokenMap["access_token"].(string)
+	brightspaceService.RefreshToken = tokenMap["refresh_token"].(string)
 	provider.AccessKey = brightspaceService.ClientSecret + ";" + brightspaceService.RefreshToken
-	if err := db.Debug().Save(&provider).Error; err != nil {
+	if err := db.Save(&provider).Error; err != nil {
+		log.Errorf("error trying to update provider access_key with new refresh token, error is %v", err)
 		return nil, err
 	}
-	//d set headers that are required for requests to brightspace
+	log.Info("refresh token updated successfully on the provider_platform")
 	headers := make(map[string]string)
 	headers["Authorization"] = "Bearer " + brightspaceService.AccessToken
 	headers["Accept"] = "application/json"
@@ -70,11 +100,13 @@ func (srv *BrightspaceService) SendPostRequest(url string, data url.Values) (*ht
 	encodedUrl := data.Encode()
 	req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(encodedUrl))
 	if err != nil {
+		log.Errorf("error creating new POST request to url %v and error is: %v", url, err)
 		return nil, err
 	}
 	req.Header.Add("Content-Type", "application/x-www-form-urlencoded") //standard header for url.Values (encoded)
 	resp, err := srv.Client.Do(req)
 	if err != nil {
+		log.Errorf("error executing POST request to url %v and error is: %v", url, err)
 		return nil, err
 	}
 	return resp, nil
@@ -83,39 +115,70 @@ func (srv *BrightspaceService) SendPostRequest(url string, data url.Values) (*ht
 func (srv *BrightspaceService) SendRequest(url string) (*http.Response, error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
+		log.Errorf("error creating new GET request to url %v and error is: %v", url, err)
 		return nil, err
 	}
 	for key, value := range *srv.BaseHeaders {
 		req.Header.Add(key, value)
 	}
 	resp, err := srv.Client.Do(req)
 	if err != nil {
+		log.Errorf("error executing GET request to url %v and error is: %v", url, err)
 		return nil, err
 	}
 	return resp, nil
 }
 
 func (srv *BrightspaceService) GetUsers(db *gorm.DB) ([]models.ImportUser, error) {
-	//get brightspace users
+	fmt.Println("GetUsers...")
 	return nil, nil
 }
 
 func (srv *BrightspaceService) ImportCourses(db *gorm.DB) error {
-	//import brightspace courses
+	pluginId, err := srv.getPluginId("Organizational Units")
+	if err != nil {
+		log.Errorf("error attempting to get plugin id for courses, error is: %v", err)
+		return err
+	}
+	log.Infof("successfully retrieved plugin id %v for downloading csv file for courses", pluginId)
+	downloadUrl := fmt.Sprintf(DataDownloadEnpoint, pluginId)
+	csvFile, err := srv.downloadAndUnzipFile("OrganizationalUnits.zip", downloadUrl)
+	if err != nil {
+		log.Errorf("error attempting to get plugin id for courses, error is: %v", err)
+		return err
+	}
+	log.Infof("successfully downloaded and unzipped %v for importing courses", csvFile)
+	bsCourses := []BrightspaceCourse{}
+	readCSV(&bsCourses, csvFile)
+	cleanUpFiles("OrganizationalUnits.zip", csvFile)
+	fields := log.Fields{"provider": srv.ProviderPlatformID, "Function": "ImportCourses", "csvFile": csvFile}
+	log.WithFields(fields).Info("importing courses from provider using csv file")
+	for _, bsCourse := range bsCourses {
+		if bsCourse.IsActive == "TRUE" && bsCourse.IsDeleted == "FALSE" && bsCourse.Type == "Course Offering" {
+			if db.Where("provider_platform_id = ? AND external_id = ?", srv.ProviderPlatformID, bsCourse.OrgUnitId).First(&models.Course{}).Error == nil {
+				continue
+			}
+			log.Infof("importing course named %v with external id %v", bsCourse.Name, bsCourse.OrgUnitId)
+			course := srv.IntoCourse(bsCourse)
+			if err := db.Create(&course).Error; err != nil {
+				log.Errorf("error creating course in db, error is: %v", err)
+				continue
+			}
+		}
+	}
 	return nil
 }
 
 func (srv *BrightspaceService) ImportMilestones(coursePair map[string]interface{}, mappings []map[string]interface{}, db *gorm.DB, lastRun time.Time) error {
-	//import milestones
+	fmt.Println("ImportMilestones...")
 	return nil
 }
 
 func (srv *BrightspaceService) ImportActivityForCourse(coursePair map[string]interface{}, db *gorm.DB) error {
-	//import activity for course
+	fmt.Println("ImportActivityForCourse...")
 	return nil
 }
 
 func (srv *BrightspaceService) GetJobParams() *map[string]interface{} {
-	//get job parameters
 	return srv.JobParams
 }