From b4bb56c05545408a0c7351320f3fb3c99e274106 Mon Sep 17 00:00:00 2001 From: Richard Salas Date: Wed, 6 Nov 2024 17:29:20 -0600 Subject: [PATCH] feat: impl ImportCourses for BrightspaceService (#479) * feat: add initial csvs local directory and default brightspace image for integration of courses * fix: modify gorm criteria for ProviderPlatform state to be enabled * feat: add Brightspace ProviderPlatformType for integration of courses * feat: add gocarina/gocvs library used for reading csv files for integration of courses * feat: add logic for the integration of brightspace courses * fix: use brightspace default img path --------- Co-authored-by: PThorpe92 --- backend/src/database/provider_platforms.go | 2 +- backend/src/models/provider_platforms.go | 1 + go.work.sum | 2 + provider-middleware/brightspace.go | 91 +++++++++++-- provider-middleware/brightspace_data.go | 151 +++++++++++++++++++-- provider-middleware/csvs/.gitkeep | 0 provider-middleware/go.mod | 1 + provider-middleware/go.sum | 2 + provider-middleware/middleware.go | 4 + 9 files changed, 226 insertions(+), 28 deletions(-) create mode 100644 provider-middleware/csvs/.gitkeep diff --git a/backend/src/database/provider_platforms.go b/backend/src/database/provider_platforms.go index db12428e..bef8f008 100644 --- a/backend/src/database/provider_platforms.go +++ b/backend/src/database/provider_platforms.go @@ -36,7 +36,7 @@ func iterMap[T any](fun func(T) T, arr []T) []T { func (db *DB) GetAllActiveProviderPlatforms() ([]models.ProviderPlatform, error) { var platforms []models.ProviderPlatform if err := db.Model(models.ProviderPlatform{}).Preload("OidcClient"). - Find(&platforms, "state = ?", "active").Error; err != nil { + Find(&platforms, "state = ?", "enabled").Error; err != nil { return nil, newGetRecordsDBError(err, "provider_platforms") } diff --git a/backend/src/models/provider_platforms.go b/backend/src/models/provider_platforms.go index 4d057d88..9de0f883 100644 --- a/backend/src/models/provider_platforms.go +++ b/backend/src/models/provider_platforms.go @@ -21,6 +21,7 @@ const ( CanvasOSS ProviderPlatformType = "canvas_oss" CanvasCloud ProviderPlatformType = "canvas_cloud" Kolibri ProviderPlatformType = "kolibri" + Brightspace ProviderPlatformType = "brightspace" ) type ProviderPlatformState string diff --git a/go.work.sum b/go.work.sum index 0f53487c..c2d3180e 100644 --- a/go.work.sum +++ b/go.work.sum @@ -18,6 +18,8 @@ github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6v github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= diff --git a/provider-middleware/brightspace.go b/provider-middleware/brightspace.go index 42e9dc07..4c320bb4 100644 --- a/provider-middleware/brightspace.go +++ b/provider-middleware/brightspace.go @@ -2,17 +2,21 @@ package main import ( "UnlockEdv2/src/models" + "encoding/json" "errors" + "fmt" "net/http" "net/url" "os" "strings" "time" + log "github.com/sirupsen/logrus" "gorm.io/gorm" ) const ( + CsvDownloadPath = "csvs" TokenEndpoint = "https://auth.brightspace.com/core/connect/token" DataSetsEndpoint = "https://unlocked.brightspacedemo.com/d2l/api/lp/1.28/dataExport/bds/list" DataDownloadEnpoint = "https://unlocked.brightspacedemo.com/d2l/api/lp/1.28/dataExport/bds/download/%s" @@ -31,8 +35,7 @@ type BrightspaceService struct { JobParams *map[string]interface{} } -// for linting reasons changed new to New below temporarily so this can be reviewed, -func NewBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, params *map[string]interface{}) (*BrightspaceService, error) { +func newBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, params *map[string]interface{}) (*BrightspaceService, error) { keysSplit := strings.Split(provider.AccessKey, ";") if len(keysSplit) < 2 { return nil, errors.New("unable to find refresh token, unable to intialize BrightspaceService") @@ -42,8 +45,14 @@ func NewBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, param return nil, errors.New("no brightspace scope found, unable to intialize BrightspaceService") } brightspaceService := BrightspaceService{ - //brightspace - set fields - JobParams: params, + ProviderPlatformID: provider.ID, + Client: &http.Client{}, + BaseURL: provider.BaseUrl, + ClientID: provider.AccountID, + ClientSecret: keysSplit[0], + RefreshToken: keysSplit[1], + Scope: scope, + JobParams: params, } data := url.Values{} data.Add("grant_type", "refresh_token") @@ -51,14 +60,35 @@ func NewBrightspaceService(provider *models.ProviderPlatform, db *gorm.DB, param data.Add("client_id", brightspaceService.ClientID) data.Add("client_secret", brightspaceService.ClientSecret) data.Add("scope", brightspaceService.Scope) - //a send post request to brightspace - //b parse response for access_token and refresh_token - //c save new refresh_token (tack it onto the end of the client secret separated by semicolon) + log.Infof("refreshing token using endpoint url %v", TokenEndpoint) + resp, err := brightspaceService.SendPostRequest(TokenEndpoint, data) + if err != nil { + log.Errorf("error sending post request to url %v", TokenEndpoint) + return nil, err + } + var tokenMap map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&tokenMap); err != nil { + log.Errorf("error decoding to response from url %v, error is: %v", TokenEndpoint, err) + return nil, err + } + if resp.StatusCode != http.StatusOK { + errType, okError := tokenMap["error"].(string) + errMsg, okDesc := tokenMap["error_description"].(string) + msg := "unable to request new refresh token from brightspace" + if okError && okDesc { + msg = fmt.Sprintf("unable to request new refresh token from brightspace, response error message is: %s: %s", errType, errMsg) + return nil, errors.New(msg) + } + return nil, errors.New(msg) + } + brightspaceService.AccessToken = tokenMap["access_token"].(string) + brightspaceService.RefreshToken = tokenMap["refresh_token"].(string) provider.AccessKey = brightspaceService.ClientSecret + ";" + brightspaceService.RefreshToken - if err := db.Debug().Save(&provider).Error; err != nil { + if err := db.Save(&provider).Error; err != nil { + log.Errorf("error trying to update provider access_key with new refresh token, error is %v", err) return nil, err } - //d set headers that are required for requests to brightspace + log.Info("refresh token updated successfully on the provider_platform") headers := make(map[string]string) headers["Authorization"] = "Bearer " + brightspaceService.AccessToken headers["Accept"] = "application/json" @@ -70,11 +100,13 @@ func (srv *BrightspaceService) SendPostRequest(url string, data url.Values) (*ht encodedUrl := data.Encode() req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(encodedUrl)) if err != nil { + log.Errorf("error creating new POST request to url %v and error is: %v", url, err) return nil, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") //standard header for url.Values (encoded) resp, err := srv.Client.Do(req) if err != nil { + log.Errorf("error executing POST request to url %v and error is: %v", url, err) return nil, err } return resp, nil @@ -83,6 +115,7 @@ func (srv *BrightspaceService) SendPostRequest(url string, data url.Values) (*ht func (srv *BrightspaceService) SendRequest(url string) (*http.Response, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { + log.Errorf("error creating new GET request to url %v and error is: %v", url, err) return nil, err } for key, value := range *srv.BaseHeaders { @@ -90,32 +123,62 @@ func (srv *BrightspaceService) SendRequest(url string) (*http.Response, error) { } resp, err := srv.Client.Do(req) if err != nil { + log.Errorf("error executing GET request to url %v and error is: %v", url, err) return nil, err } return resp, nil } func (srv *BrightspaceService) GetUsers(db *gorm.DB) ([]models.ImportUser, error) { - //get brightspace users + fmt.Println("GetUsers...") return nil, nil } func (srv *BrightspaceService) ImportCourses(db *gorm.DB) error { - //import brightspace courses + pluginId, err := srv.getPluginId("Organizational Units") + if err != nil { + log.Errorf("error attempting to get plugin id for courses, error is: %v", err) + return err + } + log.Infof("successfully retrieved plugin id %v for downloading csv file for courses", pluginId) + downloadUrl := fmt.Sprintf(DataDownloadEnpoint, pluginId) + csvFile, err := srv.downloadAndUnzipFile("OrganizationalUnits.zip", downloadUrl) + if err != nil { + log.Errorf("error attempting to get plugin id for courses, error is: %v", err) + return err + } + log.Infof("successfully downloaded and unzipped %v for importing courses", csvFile) + bsCourses := []BrightspaceCourse{} + readCSV(&bsCourses, csvFile) + cleanUpFiles("OrganizationalUnits.zip", csvFile) + fields := log.Fields{"provider": srv.ProviderPlatformID, "Function": "ImportCourses", "csvFile": csvFile} + log.WithFields(fields).Info("importing courses from provider using csv file") + for _, bsCourse := range bsCourses { + if bsCourse.IsActive == "TRUE" && bsCourse.IsDeleted == "FALSE" && bsCourse.Type == "Course Offering" { + if db.Where("provider_platform_id = ? AND external_id = ?", srv.ProviderPlatformID, bsCourse.OrgUnitId).First(&models.Course{}).Error == nil { + continue + } + log.Infof("importing course named %v with external id %v", bsCourse.Name, bsCourse.OrgUnitId) + course := srv.IntoCourse(bsCourse) + if err := db.Create(&course).Error; err != nil { + log.Errorf("error creating course in db, error is: %v", err) + continue + } + } + } return nil } func (srv *BrightspaceService) ImportMilestones(coursePair map[string]interface{}, mappings []map[string]interface{}, db *gorm.DB, lastRun time.Time) error { - //import milestones + fmt.Println("ImportMilestones...") return nil } func (srv *BrightspaceService) ImportActivityForCourse(coursePair map[string]interface{}, db *gorm.DB) error { - //import activity for course + fmt.Println("ImportActivityForCourse...") return nil } func (srv *BrightspaceService) GetJobParams() *map[string]interface{} { - //get job parameters return srv.JobParams } diff --git a/provider-middleware/brightspace_data.go b/provider-middleware/brightspace_data.go index d3a7734d..853c3efc 100644 --- a/provider-middleware/brightspace_data.go +++ b/provider-middleware/brightspace_data.go @@ -3,12 +3,18 @@ package main import ( "UnlockEdv2/src/models" "archive/zip" + "bytes" "encoding/json" + "errors" "fmt" "io" + "mime/multipart" "net/http" "os" "path/filepath" + + "github.com/gocarina/gocsv" + log "github.com/sirupsen/logrus" ) type DataSetPlugin struct { @@ -49,15 +55,97 @@ type BrightspaceEnrollment struct { EnrollmentType string `csv:"EnrollmentType"` } -func (kc *BrightspaceService) IntoImportUser(bsUser BrightspaceUser) *models.ImportUser { +func (srv *BrightspaceService) IntoImportUser(bsUser BrightspaceUser) *models.ImportUser { return nil } -func (kc *BrightspaceService) IntoCourse(bsCourse BrightspaceCourse) *models.Course { - return nil +func (srv *BrightspaceService) IntoCourse(bsCourse BrightspaceCourse) *models.Course { + id := bsCourse.OrgUnitId + courseImageUrl := fmt.Sprintf(srv.BaseURL+"/d2l/api/lp/1.28/courses/%s/image", id) + response, err := srv.SendRequest(courseImageUrl) + if err != nil { + log.Errorf("error executing request to retrieve image from url %v, error is %v", courseImageUrl, err) + return nil + } + defer response.Body.Close() + var imgPath string + var imgBytes []byte + if response.StatusCode == http.StatusOK { + imgBytes, err = io.ReadAll(response.Body) + if err != nil { + imgPath = "/brightspace.jpg" + } else { + imgPath, err = UploadBrightspaceImage(imgBytes, id) + if err != nil { + log.Errorf("error during upload of Brightspace image to UnlockEd, id used was: %v error is %v", id, err) + imgPath = "/brightspace.png" + } + } + } + course := models.Course{ + ProviderPlatformID: srv.ProviderPlatformID, + ExternalID: bsCourse.OrgUnitId, + Name: bsCourse.Name, + OutcomeTypes: "completion", + ThumbnailURL: imgPath, + Type: "fixed_enrollment", //open to discussion + Description: "Brightspace Managed Course: " + bsCourse.Name, //WIP + TotalProgressMilestones: uint(0), //WIP + ExternalURL: srv.BaseURL, //WIP + } + return &course } -func (srv *BrightspaceService) GetPluginId(pluginName string) (string, error) { +func UploadBrightspaceImage(imgBytes []byte, bsCourseId string) (string, error) { + filename := "image_brightspace" + "/" + bsCourseId + ".jpg" + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", filename) + if err != nil { + log.Errorf("error creating form file using file %v, error is %v", filename, err) + return "", err + } + if _, err = part.Write(imgBytes); err != nil { + log.Errorf("error writing bytes to mulitpart form, error is %v", err) + return "", err + } + err = writer.Close() + if err != nil { + log.Errorf("error closing file, error is %v", err) + return "", err + } + uploadEndpointUrl := os.Getenv("APP_URL") + "/upload" + request, err := http.NewRequest(http.MethodPost, uploadEndpointUrl, body) + if err != nil { + log.Errorf("error creating new POST request to url %v and error is: %v", uploadEndpointUrl, err) + return "", err + } + request.Header.Set("Content-Type", writer.FormDataContentType()) + request.Header.Set("Content-Length", fmt.Sprintf("%d", len(body.Bytes()))) + client := &http.Client{} + response, err := client.Do(request) + if err != nil { + log.Errorf("error executing POST request to url %v and error is: %v", uploadEndpointUrl, err) + return "", err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("server returned non-OK status: %s", response.Status) + } + urlRes := struct { + Data struct { + Url string `json:"url"` + } + Message string `json:"message"` + }{} + err = json.NewDecoder(response.Body).Decode(&urlRes) + if err != nil { + return "", err + } + return urlRes.Data.Url, nil +} + +func (srv *BrightspaceService) getPluginId(pluginName string) (string, error) { var pluginId string resp, err := srv.SendRequest(DataSetsEndpoint) if err != nil { @@ -66,10 +154,12 @@ func (srv *BrightspaceService) GetPluginId(pluginName string) (string, error) { defer resp.Body.Close() pluginData := []DataSetPlugin{} if err = json.NewDecoder(resp.Body).Decode(&pluginData); err != nil { + log.Errorf("error decoding to response from url %v, error is: %v", DataSetsEndpoint, err) return pluginId, err } for _, plugin := range pluginData { if plugin.Name == pluginName { + log.Infof("found plugin named %v with id %v", plugin.Name, plugin.PluginId) pluginId = plugin.PluginId break } //end if @@ -77,57 +167,92 @@ func (srv *BrightspaceService) GetPluginId(pluginName string) (string, error) { return pluginId, nil } -func (srv *BrightspaceService) DownloadAndUnzipFile(targetDirectory string, targetFileName string, endpointUrl string) (string, error) { - //initial method for download/unzip file--WIP +func readCSV[T any](values *T, csvFilePath string) { + coursesFile, err := os.OpenFile(csvFilePath, os.O_RDWR|os.O_CREATE, os.ModePerm) + if err != nil { + log.Errorf("error opening file %v, error is: %v", csvFilePath, err) + return + } + defer coursesFile.Close() + if err := gocsv.UnmarshalFile(coursesFile, values); err != nil { + log.Errorf("error parsing csv file %v into values type file, error is: %v", csvFilePath, err) + } +} + +func (srv *BrightspaceService) downloadAndUnzipFile(targetFileName string, endpointUrl string) (string, error) { var destPath string resp, err := srv.SendRequest(endpointUrl) if err != nil { return destPath, err } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + log.Errorf("unable to download resource, response returned by brightspace request url %v was %v", endpointUrl, resp.StatusCode) + return destPath, errors.New("unable to download plugin csv resource") + } if resp.StatusCode == http.StatusOK { - zipFilePath := filepath.Join(targetDirectory, targetFileName) + log.Infof("succesful request to url %v for downloading file %v", endpointUrl, targetFileName) + zipFilePath := filepath.Join(CsvDownloadPath, targetFileName) file, err := os.Create(zipFilePath) if err != nil { + log.Errorf("error creating file %v used to write csv data into, error is: %v", zipFilePath, err) return destPath, err } _, err = io.Copy(file, resp.Body) if err != nil { + log.Errorf("error writing response to file %v, error is: %v", zipFilePath, err) return destPath, err } - file.Close() - zipFile, err := zip.OpenReader(zipFilePath) //open the zip file + err = file.Close() + if err != nil { + log.Errorf("error closing file %v, error is: %v", zipFilePath, err) + } + zipFile, err := zip.OpenReader(zipFilePath) if err != nil { + log.Errorf("error opening zip file reader for file %v, error is: %v", zipFilePath, err) return destPath, err } defer zipFile.Close() //close it later for _, zippedFile := range zipFile.File { - destPath = filepath.Join(targetDirectory, zippedFile.Name) - if zippedFile.FileInfo().IsDir() { + destPath = filepath.Join(CsvDownloadPath, zippedFile.Name) + if zippedFile.FileInfo().IsDir() { //handles all zipped files if err := os.MkdirAll(destPath, os.ModePerm); err != nil { - fmt.Println("error occurred while trying to make directories, error is: ", err) + log.Errorf("error making directory to destination path %v, error is: %v", destPath, err) } continue } - //there is going to be no directory for this file, as these are csv files if err = os.MkdirAll(filepath.Dir(destPath), os.ModePerm); err != nil { + log.Errorf("error making directory to destination path %v, error is: %v", destPath, err) return destPath, err } outFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zippedFile.Mode()) if err != nil { + log.Errorf("error openings %v, error is: %v", destPath, err) return destPath, err } defer outFile.Close() rc, err := zippedFile.Open() if err != nil { + log.Errorf("error opening zipped file csv file %v, error is: %v", zippedFile.Name, err) return destPath, err } defer rc.Close() _, err = io.Copy(outFile, rc) if err != nil { + log.Errorf("error copying zipped file %v to destination file %v, error is: %v", zippedFile.Name, destPath, err) return destPath, err } } } return destPath, err } + +func cleanUpFiles(zipFileName, csvFile string) { + zipFilePath := filepath.Join(CsvDownloadPath, zipFileName) + if err := os.Remove(zipFilePath); err != nil { + log.Warnf("unable to delete file %v", zipFilePath) + } + if err := os.Remove(csvFile); err != nil { + log.Warnf("unable to delete file %v", csvFile) + } +} diff --git a/provider-middleware/csvs/.gitkeep b/provider-middleware/csvs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/provider-middleware/go.mod b/provider-middleware/go.mod index 0b9e94b0..eb027cc6 100644 --- a/provider-middleware/go.mod +++ b/provider-middleware/go.mod @@ -13,6 +13,7 @@ require ( require ( github.com/cockroachdb/apd v1.1.0 // indirect + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect diff --git a/provider-middleware/go.sum b/provider-middleware/go.sum index c3a5d83b..b32ad215 100644 --- a/provider-middleware/go.sum +++ b/provider-middleware/go.sum @@ -3,6 +3,8 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGUK1IDqRa5lAAvEkZG1LKaCRc= diff --git a/provider-middleware/middleware.go b/provider-middleware/middleware.go index a75b4e7a..3ae6af89 100644 --- a/provider-middleware/middleware.go +++ b/provider-middleware/middleware.go @@ -31,6 +31,8 @@ func (sh *ServiceHandler) initServiceFromRequest(ctx context.Context, r *http.Re return NewKolibriService(&provider, nil), nil case models.CanvasCloud, models.CanvasOSS: return newCanvasService(&provider, nil), nil + case models.Brightspace: + return newBrightspaceService(&provider, sh.db, nil) } return nil, fmt.Errorf("unsupported provider type: %s", provider.Type) } @@ -62,6 +64,8 @@ func (sh *ServiceHandler) initProviderPlatformService(ctx context.Context, msg * return NewKolibriService(&provider, &body), nil case models.CanvasCloud, models.CanvasOSS: return newCanvasService(&provider, &body), nil + case models.Brightspace: + return newBrightspaceService(&provider, sh.db, &body) } return nil, fmt.Errorf("unsupported provider type: %s", provider.Type) }