diff --git a/README.md b/README.md index a53a005..7789e47 100644 --- a/README.md +++ b/README.md @@ -1 +1,67 @@ -# twitter-saver \ No newline at end of file +# twitter-saver + +## Description + +### Info +The program was created for *Team project - development of data analysis system* course run by [@pbiecek](https://github.com/pbiecek) at Warsaw University of Technology. + +### Program description + +The aim of the created program is to download tweets from twitter by defined user or keyword. Downloading can be done in two modes: + +* stream +* history + +### Further information + +Full specification and more detailed description of summarization features (in Polish) can be found in [this file](https://github.com/minorczyka/twitter-saver/blob/master/docs/Instrukcja%20obs%C5%82ugi.pdf). + +## Installation + +1. Download binary file from latest release +2. Prepare `config.yaml` +3. Run binaries `twitter-saver` and `web` + +## Usage + +### Running + +The programs can be run from command line with following arguments: + +* `--config` - path to config file + +Config file is stored in YAML format. It contains following information: + +- `db`: + - `host` + - `port` + - `user` + - `password` + - `dbName` - database name in which data will be stored + - `sslMode` - `enable` or `disable` +- `web` - web interface parameters: + - `port` - port on which server will be working + - `secret` - private key used to sign session identifiers. Should be random and renewed periodically. Keys shorter than 256 bits are not recommended. +- `users` - sequence of user accounts. Each account consists of: + - `username` + - `password` +- `twitter` - twitter API keys: + - `consumerKey` + - `consumerSecret` + - `token` + - `tokenSecret` +- `json` - defines additional fields from tweet saved in database + - `all` - saves whole tweet content + - `fields` - sequence of field names to be stored +- `autoDeleteDays` - number of days after which data will be automatically removed + +### Screenshots and live examples + +Screenshot of project UI: +![the screenshot](https://github.com/minorczyka/twitter-saver/blob/master/misc/screenshots/screen1.png) + +## Authors + +* Piotr Krzeszewski +* Łukasz Ławniczak +* Artur Minorczyk diff --git a/core/config.go b/core/config.go new file mode 100644 index 0000000..5db809b --- /dev/null +++ b/core/config.go @@ -0,0 +1,97 @@ +package core + +import ( + "errors" + "fmt" + + "github.com/dghubble/go-twitter/twitter" + "github.com/dghubble/oauth1" +) + +type DbConfig struct { + Host *string + Port *string + User *string + Password *string + DbName *string `yaml:"dbName"` + SslMode *string `yaml:"sslMode"` +} + +type TwitterConfig struct { + ConsumerKey *string `yaml:"consumerKey"` + ConsumerSecret *string `yaml:"consumerSecret"` + Token *string + TokenSecret *string `yaml:"tokenSecret"` +} + +type JsonConfig struct { + All bool + Fields []string +} + +type UserAccount struct { + Username *string + Password *string +} + +type WebConfig struct { + Port *int + Secret *string + Users []UserAccount +} + +func (d *DbConfig) ConnectionString() (string, error) { + if d.Host == nil { + return "", errors.New("Database host not specified.") + } + if d.Port == nil { + return "", errors.New("Database port not specified.") + } + if d.User == nil { + return "", errors.New("Database user not specified.") + } + if d.Password == nil { + return "", errors.New("Database password not specified.") + } + if d.DbName == nil { + return "", errors.New("Database name not specified.") + } + if d.SslMode == nil { + return "", errors.New("Database ssl mode not specified.") + } + return fmt.Sprintf("host=%s port=%s user=%s dbname=%s password=%s sslmode=%s", + *d.Host, *d.Port, *d.User, *d.DbName, *d.Password, *d.SslMode), nil +} + +func (t *TwitterConfig) TwitterClient() (*twitter.Client, error) { + if t.ConsumerKey == nil { + return nil, errors.New("Twitter consumer key not specified.") + } + if t.ConsumerSecret == nil { + return nil, errors.New("Twitter consumer secret not specified.") + } + if t.Token == nil { + return nil, errors.New("Twitter token not specified.") + } + if t.TokenSecret == nil { + return nil, errors.New("Twitter token secret not specified.") + } + config := oauth1.NewConfig(*t.ConsumerKey, *t.ConsumerSecret) + token := oauth1.NewToken(*t.Token, *t.TokenSecret) + httpClient := config.Client(oauth1.NoContext, token) + return twitter.NewClient(httpClient), nil +} + +func (w *WebConfig) WebAccounts() (map[string]string, error) { + result := make(map[string]string) + for _, user := range w.Users { + if user.Username == nil { + return nil, errors.New("Username not specified for account.") + } + if user.Password == nil { + return nil, errors.New("Password not specified for account.") + } + result[*user.Username] = *user.Password + } + return result, nil +} diff --git a/core/connect.go b/core/connect.go new file mode 100644 index 0000000..f63ab6c --- /dev/null +++ b/core/connect.go @@ -0,0 +1,22 @@ +package core + +import ( + "log" + + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" +) + +func Connect(connectionString string) *gorm.DB { + db, err := gorm.Open("postgres", connectionString) + if err != nil { + log.Fatal(err) + } + + db.AutoMigrate(&Tweet{}) + db.AutoMigrate(&Object{}) + + db.Model(&Tweet{}).AddForeignKey("object_id", "objects(id)", "CASCADE", "NO ACTION") + + return db +} diff --git a/core/models.go b/core/models.go new file mode 100644 index 0000000..4687fe2 --- /dev/null +++ b/core/models.go @@ -0,0 +1,44 @@ +package core + +import ( + "time" +) + +type Tweet struct { + ID uint `gorm:"primary_key"` + CreatedAt time.Time + + TweetId int64 `gorm:"not null"` + PublishedAt time.Time `gorm:"not null"` + UserId int64 `gorm:"not null"` + Text string `gorm:"not null" sql:"index"` + ExtendedInfo string + ObjectId uint +} + +type ObjectSource int32 + +const ( + HistorySource ObjectSource = iota + 1 + StreamSource +) + +type ObjectType = int32 + +const ( + UserType ObjectType = iota + 1 + KeywordType +) + +type Object struct { + ID uint `gorm:"primary_key"` + CreatedAt time.Time + DeletedAt *time.Time `sql:"index"` + + Source ObjectSource `gorm:"not null"` + Type ObjectType `gorm:"not null"` + Query string `gorm:"not null"` + HistoryFrom *time.Time + HistoryDone bool + Tweets []Tweet +} diff --git a/core/object_dao.go b/core/object_dao.go new file mode 100644 index 0000000..631cacd --- /dev/null +++ b/core/object_dao.go @@ -0,0 +1,33 @@ +package core + +import ( + "github.com/jinzhu/gorm" +) + +func FindStreamObjects(db *gorm.DB) []Object { + var objects []Object + db.Where(&Object{Source: StreamSource}).Find(&objects) + return objects +} + +func CountStreamObjects(db *gorm.DB) int { + var count int + db.Model(&Object{}).Where("source = ?", StreamSource).Count(&count) + return count +} + +func FindHistoryObjects(db *gorm.DB) []Object { + var objects []Object + db.Where(&Object{Source: HistorySource}).Find(&objects) + return objects +} + +func FindAllObjects(db *gorm.DB) []Object { + var objects []Object + db.Unscoped().Find(&objects) + return objects +} + +func UpdateObjectHistory(db *gorm.DB, object *Object, history bool) { + db.Model(object).Update("history_done", history) +} diff --git a/core/tweet_dao.go b/core/tweet_dao.go new file mode 100644 index 0000000..ccac647 --- /dev/null +++ b/core/tweet_dao.go @@ -0,0 +1,136 @@ +package core + +import ( + "encoding/json" + "errors" + "strings" + "time" + + "github.com/dghubble/go-twitter/twitter" + "github.com/jinzhu/gorm" +) + +func addField(jsonMap map[string]interface{}, result map[string]interface{}, field string) error { + index := strings.Index(field, ".") + if index > -1 { + key := field[:index] + rest := field[index+1:] + value, prs := jsonMap[key] + if !prs { + return errors.New("Field " + key + " not present in json") + } + v, ok := value.(map[string]interface{}) + if !ok { + return errors.New("Field " + key + " is not nested value in json") + } + + r, prs := result[key] + if !prs { + r = make(map[string]interface{}) + } + + res := r.(map[string]interface{}) + err := addField(v, res, rest) + if err != nil { + return err + } + result[key] = res + return nil + } else { + value, prs := jsonMap[field] + if !prs { + return errors.New("Field " + field + " not present in json") + } + result[field] = value + return nil + } +} + +func jsonWithFields(j []byte, fields []string) ([]byte, error) { + var jsonMap map[string]interface{} + err := json.Unmarshal(j, &jsonMap) + if err != nil { + return nil, err + } + + result := make(map[string]interface{}) + for _, field := range fields { + err := addField(jsonMap, result, field) + if err != nil { + return nil, err + } + } + + resultBytes, err := json.Marshal(result) + if err != nil { + return nil, err + } + return resultBytes, nil +} + +func InsertTweet(db *gorm.DB, tweet *twitter.Tweet, objectId uint, fields []string, allFields bool) error { + publishedAt, err := time.Parse(time.RubyDate, tweet.CreatedAt) + if err != nil { + return err + } + + entity := &Tweet{ + PublishedAt: publishedAt, + TweetId: tweet.ID, + UserId: tweet.User.ID, + Text: tweet.Text, + ObjectId: objectId, + } + + if allFields || len(fields) > 0 { + j, err := json.Marshal(tweet) + if err != nil { + db.Create(&entity) + return err + } + + if allFields { + entity.ExtendedInfo = string(j) + } else { + res, err := jsonWithFields(j, fields) + if err != nil { + db.Create(&entity) + return err + } + entity.ExtendedInfo = string(res) + } + } + + db.Create(&entity) + return nil +} + +func InsertTweetUnique(db *gorm.DB, tweet *twitter.Tweet, objectId uint, fields []string, allFields bool) error { + count := 0 + db.Model(&Tweet{}).Where("tweet_id = ? AND object_id = ?", tweet.ID, objectId).Count(&count) + if count == 0 { + return InsertTweet(db, tweet, objectId, fields, allFields) + } else { + return nil + } +} + +func RemoveTweetsOlderThan(db *gorm.DB, date time.Time) error { + _, err := db.Raw("DELETE FROM tweets WHERE published_at::date < ?", date).Rows() + return err +} + +func LatestTweetForObject(db *gorm.DB, objectId uint) (int64, error) { + rows, err := db.Raw("SELECT MAX(tweet_id) FROM tweets WHERE object_id = ?", objectId).Rows() + if err != nil { + return 0, err + } + + rows.Next() + var tweetId int64 + err = rows.Scan(&tweetId) + if err != nil { + return 0, err + } + return tweetId, nil +} diff --git "a/docs/Instrukcja obs\305\202ugi.pdf" "b/docs/Instrukcja obs\305\202ugi.pdf" new file mode 100644 index 0000000..1a009c6 Binary files /dev/null and "b/docs/Instrukcja obs\305\202ugi.pdf" differ diff --git a/front/.gitignore b/front/.gitignore new file mode 100644 index 0000000..3dadcdf --- /dev/null +++ b/front/.gitignore @@ -0,0 +1,8 @@ +target/ +project/plugins/project/ +*.class + +*.iml +*.ipr +*.iws +.idea/ diff --git a/front/build.sbt b/front/build.sbt new file mode 100644 index 0000000..42e3b8f --- /dev/null +++ b/front/build.sbt @@ -0,0 +1,24 @@ +enablePlugins(ScalaJSPlugin) + +name := "twitter" + +version := "0.1" + +scalaVersion := "2.12.4" + +scalaJSUseMainModuleInitializer := true + +libraryDependencies ++= Seq( + "com.github.japgolly.scalajs-react" %%% "core" % "1.1.0", + "com.github.japgolly.scalajs-react" %%% "extra" % "1.1.0", + "io.suzaku" %%% "diode" % "1.1.3", + "io.suzaku" %%% "diode-react" % "1.1.3", + "io.circe" %%% "circe-core" % "0.9.1", + "io.circe" %%% "circe-generic" % "0.9.1", + "io.circe" %%% "circe-parser" % "0.9.1" +) + +jsDependencies ++= Seq( + "org.webjars.bower" % "react" % "15.6.1" / "react-with-addons.js" minified "react-with-addons.min.js" commonJSName "React", + "org.webjars.bower" % "react" % "15.6.1" / "react-dom.js" minified "react-dom.min.js" dependsOn "react-with-addons.js" commonJSName "ReactDOM" +) diff --git a/front/dev-server.sh b/front/dev-server.sh new file mode 100644 index 0000000..11910f8 --- /dev/null +++ b/front/dev-server.sh @@ -0,0 +1,2 @@ +cd target/scala-2.12 +live-server --watch=twitter-fastopt.js,twitter-jsdeps.js,classes/index.html --open=classes/index-dev.html diff --git a/front/project/build.properties b/front/project/build.properties new file mode 100644 index 0000000..5517665 --- /dev/null +++ b/front/project/build.properties @@ -0,0 +1 @@ +sbt.version = 1.1.1 \ No newline at end of file diff --git a/front/project/plugins.sbt b/front/project/plugins.sbt new file mode 100644 index 0000000..de824ea --- /dev/null +++ b/front/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.22") diff --git a/front/src/main/resources/app.css b/front/src/main/resources/app.css new file mode 100644 index 0000000..2f41639 --- /dev/null +++ b/front/src/main/resources/app.css @@ -0,0 +1,53 @@ +body { + min-height: 100vh; + background-color: #f9f9f9; +} + +section.app { + min-height: 100vh; +} + +div.main { + display: flex; + min-height: 100vh; + flex-direction: column; +} + +main { + flex: 1 0 auto; +} + +.page-footer { + padding: 0; +} + +.login-panel { + margin-top: 60px; + padding-top: 1px; + padding-bottom: 1px; +} + +.loading { + margin: 50px; +} + +h5.title { + margin-top: 0; +} + +.radio-group { + margin-top: 8px; + margin-bottom: 8px; +} + +.radio-group label { + margin-right: 8px; +} + +.no-padding-left { + padding-left: 0 !important; +} + +.no-padding-right { + padding-right: 0 !important; +} diff --git a/front/src/main/resources/index-dev.html b/front/src/main/resources/index-dev.html new file mode 100644 index 0000000..2e972a8 --- /dev/null +++ b/front/src/main/resources/index-dev.html @@ -0,0 +1,21 @@ + + + + + Twitter Saver + + + + + + + + + +
+ + + + + + diff --git a/front/src/main/resources/index.html b/front/src/main/resources/index.html new file mode 100644 index 0000000..904cc66 --- /dev/null +++ b/front/src/main/resources/index.html @@ -0,0 +1,21 @@ + + + + + Twitter Saver + + + + + + + + + +
+ + + + + + diff --git a/front/src/main/scala/twitter/AppModel.scala b/front/src/main/scala/twitter/AppModel.scala new file mode 100644 index 0000000..480de60 --- /dev/null +++ b/front/src/main/scala/twitter/AppModel.scala @@ -0,0 +1,25 @@ +package twitter + +import diode.Circuit +import diode.react.ReactConnector +import twitter.login.{LoginHandler, LoginModel} +import twitter.tweets.{TweetsHandler, TweetsModel} + +case class AppModel(models: Models) + +case class Models(loginModel: LoginModel, tweetsModel: TweetsModel) + +object AppCircuit extends Circuit[AppModel] with ReactConnector[AppModel] { + + override protected def initialModel: AppModel = AppModel( + Models( + LoginModel.initModel, + TweetsModel.initModel + ) + ) + + override protected def actionHandler: AppCircuit.HandlerFunction = composeHandlers( + new LoginHandler(zoomTo(_.models.loginModel)), + new TweetsHandler(zoomTo(_.models.tweetsModel)) + ) +} \ No newline at end of file diff --git a/front/src/main/scala/twitter/MainApp.scala b/front/src/main/scala/twitter/MainApp.scala new file mode 100644 index 0000000..8b68a3a --- /dev/null +++ b/front/src/main/scala/twitter/MainApp.scala @@ -0,0 +1,20 @@ +package twitter + +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import org.scalajs.dom +import twitter.login.TryInitialLogin +import twitter.router.RouterView + +object MainApp { + + val apiUrl = "/api" + + def main(args: Array[String]): Unit = { + val modelsConnection = AppCircuit.connect(_.models) + val routerView = modelsConnection(proxy => RouterView(proxy)) + routerView.renderIntoDOM(dom.document.getElementsByClassName("app")(0).domAsHtml) + + AppCircuit.dispatch(TryInitialLogin) + } +} diff --git a/front/src/main/scala/twitter/login/LoginModel.scala b/front/src/main/scala/twitter/login/LoginModel.scala new file mode 100644 index 0000000..e60b9c5 --- /dev/null +++ b/front/src/main/scala/twitter/login/LoginModel.scala @@ -0,0 +1,71 @@ +package twitter.login + +import diode._ +import org.scalajs.dom +import twitter.login.LoginRest.SuccessfulLogin +import twitter.tweets.{ClearCredentials, LoginSuccessful} + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.scalajs.js + +case class LoginModel(username: String, password: String, progress: Boolean, error: Option[String]) + +object LoginModel { + val initModel = LoginModel("", "", progress = false, None) +} + +case class UsernameChanged(username: String) extends Action +case class PasswordChanged(username: String) extends Action +case object LoginSubmit extends Action +case class LoginError(error: String) extends Action +case object Logout extends Action +case object TryInitialLogin extends Action + +class LoginHandler[M](modelRW: ModelRW[M, LoginModel]) extends ActionHandler(modelRW) { + private val credentialsKey = "credentials" + + private def storeCredentials(credentials: String): Unit = + dom.window.localStorage.setItem(credentialsKey, credentials) + + private def loadCredentials(): Option[String] = { + val credentials = dom.window.localStorage.getItem(credentialsKey) + if (credentials == null || credentials == "null") None else Some(credentials) + } + + override def handle = { + case UsernameChanged(u) => + updated(value.copy(username = u)) + case PasswordChanged(p) => + updated(value.copy(password = p)) + case LoginError(e) => + updated(value.copy(progress = false, error = Some(e))) + case LoginSubmit => + val effect = Effect(LoginRest.tryLogin(value.username, value.password) + .map { + case Left(e) => LoginError(e.getMessage) + case Right(SuccessfulLogin(token)) => + val credentials = s"Bearer $token" + storeCredentials(credentials) + LoginSuccessful(credentials) + }) + updated(value.copy(progress = true), effect) + case Logout => + storeCredentials(null) + updated(LoginModel.initModel, Effect(Future.successful(ClearCredentials))) + case TryInitialLogin => + loadCredentials() match { + case Some(credentials) => + val effect = Effect(LoginRest.tryRefreshToken(credentials) + .map { + case Left(_) => NoAction + case Right(SuccessfulLogin(token)) => + val credentials = s"Bearer $token" + storeCredentials(credentials) + LoginSuccessful(credentials) + }) + updated(value, effect) + case None => noChange + } + } +} diff --git a/front/src/main/scala/twitter/login/LoginRest.scala b/front/src/main/scala/twitter/login/LoginRest.scala new file mode 100644 index 0000000..bb20df9 --- /dev/null +++ b/front/src/main/scala/twitter/login/LoginRest.scala @@ -0,0 +1,47 @@ +package twitter.login + +import io.circe.generic.auto._ +import io.circe.parser.decode +import io.circe.syntax._ +import org.scalajs.dom.ext.Ajax.InputData +import org.scalajs.dom.ext.{Ajax, AjaxException} +import twitter.MainApp + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +object LoginRest { + + case class MessageInfo(message: String) + + case class LoginInfo(username: String, password: String) + + case class SuccessfulLogin(token: String) + + def authorizationHeader(credentials: String): Map[String, String] = + Map("Authorization" -> credentials) + + implicit class AjaxFuture[A](val value: Future[Either[Exception, A]]) extends AnyVal { + def recoverAjax(): Future[Either[Exception, A]] = value.recover { + case AjaxException(xhr) if xhr.status == 401 => Left(new Exception("Nieprawidłowa nazwa użytkownika lub hasło.")) + case AjaxException(xhr) if xhr.status == 406 => Left(new Exception("Przekroczono dozwoloną liczbę obiektów tego typu.")) + case AjaxException(_) => Left(new Exception(s"Błąd połączenia")) + case e: Exception => Left(e) + } + } + + def tryLogin(username: String, password: String): Future[Either[Exception, SuccessfulLogin]] = { + val url = s"${MainApp.apiUrl}/login" + val json = LoginInfo(username, password).asJson.noSpaces + Ajax.post(url, data = InputData.str2ajax(json)) + .map(r => decode[SuccessfulLogin](r.responseText)) + .recoverAjax() + } + + def tryRefreshToken(credentials: String): Future[Either[Exception, SuccessfulLogin]] = { + val url = s"${MainApp.apiUrl}/refresh_token" + Ajax.get(url, headers = authorizationHeader(credentials)) + .map(r => decode[SuccessfulLogin](r.responseText)) + .recoverAjax() + } +} diff --git a/front/src/main/scala/twitter/login/LoginView.scala b/front/src/main/scala/twitter/login/LoginView.scala new file mode 100644 index 0000000..4c2a0c4 --- /dev/null +++ b/front/src/main/scala/twitter/login/LoginView.scala @@ -0,0 +1,65 @@ +package twitter.login + +import diode.Action +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ + +object LoginView { + + case class Props(loginModel: LoginModel, dispatch: Action => Callback) + + class Backend($: BackendScope[Props, Unit]) { + private def submit(dispatch: Action => Callback)(e: ReactEvent) = e.preventDefaultCB >> + dispatch(LoginSubmit) >> + CallbackTo.pure(false) + + def render(p: Props): VdomElement = { + <.div(^.className := "row", + <.div(^.className := "col s6 offset-s3", + <.div(^.className := "card-panel login-panel", + <.form(^.onSubmit ==> submit(p.dispatch), + <.div(^.className := "row", + <.div(^.className := "col s12") + ), + <.div(^.className := "row", + <.div(^.className := "input-field col s12", + <.input(^.className := "validate", ^.`type` := "text", ^.name := "username", ^.id := "username", + ^.required := true, ^.disabled := p.loginModel.progress, ^.onChange ==> { e: ReactEventFromInput => + p.dispatch(UsernameChanged(e.target.value)) + }), + <.label(^.`for` := "username", "Nazwa użytkownika") + ) + ), + <.div(^.className := "row", + <.div(^.className := "input-field col s12", + <.input(^.className := "validate", ^.`type` := "password", ^.name := "password", ^.id := "password", + ^.required := true, ^.disabled := p.loginModel.progress, ^.onChange ==> { e: ReactEventFromInput => + p.dispatch(PasswordChanged(e.target.value)) + }), + <.label(^.`for` := "password", "Hasło") + ) + ), + p.loginModel.error.whenDefined { e => + <.div(^.className := "row", + <.p(^.className := "red-text", e) + ) + }, + <.div(^.className := "row", + <.button(^.className := "col s12 btn btn-large waves-effect", ^.`type` := "submit", + ^.name := "btn_login", ^.disabled := p.loginModel.progress, "Zaloguj") + ) + ) + ) + ) + ) + } + } + + val component = ScalaComponent.builder[Props]("LoginView") + .stateless + .renderBackend[Backend] + .build + + def apply(loginModel: LoginModel, dispatch: Action => Callback) = + component(Props(loginModel, dispatch)) +} diff --git a/front/src/main/scala/twitter/router/RouterView.scala b/front/src/main/scala/twitter/router/RouterView.scala new file mode 100644 index 0000000..fdc5c80 --- /dev/null +++ b/front/src/main/scala/twitter/router/RouterView.scala @@ -0,0 +1,64 @@ +package twitter.router + +import diode.react.ModelProxy +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import twitter.Models +import twitter.login.{LoginView, Logout} +import twitter.tweets.TweetsView + +object RouterView { + + case class Props(proxy: ModelProxy[Models]) + + class Backend($: BackendScope[Props, Unit]) { + + def render(p: Props): VdomElement = { + val proxy = p.proxy() + <.div(^.className := "main", + <.header( + <.nav(^.className := "teal", + <.div(^.className := "nav-wrapper container", + <.a(^.className := "brand-logo", "Twitter Saver"), + <.ul(^.className := "right", + proxy.tweetsModel.credentials match { + case Some(_) => + <.li( + <.a(^.href := "#!", ^.onClick --> p.proxy.dispatchCB(Logout), + "Wyloguj", + <.i(^.className := "material-icons right", "exit_to_app") + ) + ) + case None => EmptyVdom + } + ) + ) + ) + ), + <.main( + <.div(^.className := "container", + proxy.tweetsModel.credentials match { + case Some(_) => p.proxy.wrap(_.tweetsModel)(x => TweetsView(x(), x.dispatchCB)) + case None => p.proxy.wrap(_.loginModel)(x => LoginView(x(), x.dispatchCB)) + } + ) + ), + <.footer(^.className := "page-footer teal", + <.div(^.className := "footer-copyright", + <.div(^.className := "container", + "Piotr Krzeszewski, Łukasz Ławniczak, Artur Minorczyk © 2018 - Budowa systemu analizy danych" + ) + ) + ) + ) + } + } + + val component = ScalaComponent.builder[Props]("RouterView") + .stateless + .renderBackend[Backend] + .build + + def apply(proxy: ModelProxy[Models]) = + component(Props(proxy)) +} \ No newline at end of file diff --git a/front/src/main/scala/twitter/shared/DateUtils.scala b/front/src/main/scala/twitter/shared/DateUtils.scala new file mode 100644 index 0000000..8d0dc8c --- /dev/null +++ b/front/src/main/scala/twitter/shared/DateUtils.scala @@ -0,0 +1,42 @@ +package twitter.shared + +import scala.scalajs.js +import scala.scalajs.js.Date +import scala.util.Try + +object DateUtils { + def parseDate(value: String): Try[Date] = { + for { + day <- Try(value.substring(0, 2).toInt) + month <- Try(value.substring(3, 5).toInt) + year <- Try(value.substring(6).toInt) + } yield new Date(year, month - 1, day) + } + + def showDate(date: Date): String = { + val day = date.getDate() + val month = date.getMonth() + 1 + val year = date.getFullYear() + f"$day%02d-$month%02d-$year" + } + + def min(a: Date, b: Date): Date = if (a.getTime() < b.getTime()) a else b + + def max(a: Date, b: Date): Date = if (a.getTime() > b.getTime()) a else b + + def nowWithoutTime(): Date = { + val date = new Date() + date.setHours(0, 0, 0, 0) + date + } + + val i18n = js.Dynamic.literal( + cancel = "Anuluj", + clear = "Wyczyść", + months = js.Array("Styczeń", "Luty", "Marzec", "Kwiecień", "Maj", "Czerwiec", "Lipiec", "Sierpień", "Wrzesień", "Październik", "Listopad", "Grudzień"), + monthsShort = js.Array("Sty", "Lut", "Mar", "Kwi", "Maj", "Cze", "Lip", "Sie", "Wrz", "Paź", "Lis", "Gru"), + weekdays = js.Array("Niedziela", "Poniedziałek", "Wtorek", "Środa", "Czwartek", "Piątek", "Sobota"), + weekdaysShort = js.Array("Nie", "Pon", "Wto", "Śro", "Czw", "Pią", "Sob"), + weekdaysAbbrev = js.Array("N", "Pn", "W", "Ś", "Cz", "Pt", "S") + ) +} diff --git a/front/src/main/scala/twitter/shared/LoadingView.scala b/front/src/main/scala/twitter/shared/LoadingView.scala new file mode 100644 index 0000000..587f385 --- /dev/null +++ b/front/src/main/scala/twitter/shared/LoadingView.scala @@ -0,0 +1,34 @@ +package twitter.shared + +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ + +object LoadingView { + + case class Props() + + class Backend($: BackendScope[Props, Unit]) { + def render(p: Props): VdomElement = { + <.div(^.className := "preloader-wrapper big active", + <.div(^.className := "spinner-layer spinner-blue-only", + <.div(^.className := "circle-clipper left", + <.div(^.className := "circle") + ), + <.div(^.className := "gap-patch", + <.div(^.className := "circle") + ), + <.div(^.className := "circle-clipper right", + <.div(^.className := "circle") + ) + ) + ) + } + } + + val component = ScalaComponent.builder[Props]("LoadingView") + .stateless + .renderBackend[Backend] + .build + + def apply() = component(Props()) +} diff --git a/front/src/main/scala/twitter/shared/PaginationView.scala b/front/src/main/scala/twitter/shared/PaginationView.scala new file mode 100644 index 0000000..9f48c15 --- /dev/null +++ b/front/src/main/scala/twitter/shared/PaginationView.scala @@ -0,0 +1,45 @@ +package twitter.shared + +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ + +object PaginationView { + + case class Props(itemsCount: Int, + currentPage: Int, + pageSize: Int, + onPageChange: Int => Callback) + + class Backend($: BackendScope[Props, Unit]) { + def render(p: Props): VdomElement = { + val pageCount = (p.itemsCount + p.pageSize - 1) / p.pageSize + val first = if (p.currentPage == 0) "disabled" else "waves-effect" + val last = if (p.currentPage + 1 == pageCount) "disabled" else "waves-effect" + <.ul(^.className := "pagination", + <.li(^.className := first, + <.a(^.href := "#!", ^.onClick --> (if (p.currentPage == 0) Callback.empty else p.onPageChange(p.currentPage - 1)), + <.i(^.className := "material-icons", "chevron_left") + ) + ), + (0 until pageCount).toTagMod { i => + <.li(^.className := (if (i == p.currentPage) "active" else "waves-effect"), + ^.onClick --> (if (i == p.currentPage) Callback.empty else p.onPageChange(i)), + <.a(^.href := "#!", i + 1) + ) + }, + <.li(^.className := last, + <.a(^.href := "#!", ^.onClick --> (if (p.currentPage + 1 == pageCount) Callback.empty else p.onPageChange(p.currentPage + 1)), + <.i(^.className := "material-icons", "chevron_right") + ) + ) + ) + } + } + + val component = ScalaComponent.builder[Props]("PaginationView") + .stateless + .renderBackend[Backend] + .build + + def apply(props: Props) = component(props) +} diff --git a/front/src/main/scala/twitter/tweets/AddObjectView.scala b/front/src/main/scala/twitter/tweets/AddObjectView.scala new file mode 100644 index 0000000..1275f26 --- /dev/null +++ b/front/src/main/scala/twitter/tweets/AddObjectView.scala @@ -0,0 +1,116 @@ +package twitter.tweets + +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import org.scalajs.dom +import org.scalajs.dom.raw.HTMLInputElement +import twitter.AppCircuit +import twitter.shared.DateUtils +import twitter.tweets.TweetsRest.{HistorySource, NewObjectInfo, StreamSource, TweetSource} + +import scala.scalajs.js +import scala.scalajs.js.Date + +object AddObjectView { + + case class Props(info: NewObjectInfo, + progress: Boolean, + error: Option[String], + onQueryChange: String => Callback, + onTypeChange: String => Callback, + onSourceChange: TweetSource => Callback, + onSubmit: Callback) + + class Backend($: BackendScope[Props, Unit]) { + private def submit(onSubmit: Callback)(e: ReactEvent) = e.preventDefaultCB >> + onSubmit >> + CallbackTo.pure(false) + + def render(p: Props): VdomElement = { + val objectType = p.info.objectType + val source = p.info.source + <.div(^.className := "card", + <.div(^.className := "card-content", + <.form(^.onSubmit ==> submit(p.onSubmit), + <.h5(^.className := "title", "Nowy obiekt"), + <.div(^.className := "radio-group", + <.label( + <.input(^.`type` := "radio", ^.checked := objectType == NewObjectInfo.keywordType, ^.disabled := p.progress, + ^.onChange --> p.onTypeChange(NewObjectInfo.keywordType)), + <.span("Słowo kluczowe") + ), + <.label( + <.input(^.`type` := "radio", ^.checked := objectType == NewObjectInfo.userType, ^.disabled := p.progress, + ^.onChange --> p.onTypeChange(NewObjectInfo.userType)), + <.span("Użytkownik") + ) + ), + <.div(^.className := "input-field", + <.input(^.`type` := "text", ^.id := "query", ^.required := true, ^.disabled := p.progress, + ^.value := p.info.query, ^.onChange ==> { e: ReactEventFromInput => + p.onQueryChange(e.target.value) + }), + <.label(^.`for` := "query", if (objectType == NewObjectInfo.keywordType) "Słowo kluczowe" else "Użytkownik") + ), + <.div(^.className := "radio-group", + <.label( + <.input(^.`type` := "radio", ^.checked := source == StreamSource, ^.disabled := p.progress, + ^.onChange --> p.onSourceChange(StreamSource)), + <.span("Stream") + ), + <.label( + <.input(^.`type` := "radio", ^.checked := source != StreamSource, ^.disabled := p.progress, + ^.onChange --> p.onSourceChange(HistorySource(DateUtils.nowWithoutTime()))), + <.span("Batch") + ) + ), + p.info.source match { + case HistorySource(date) => + <.div(^.className := "input-field", + <.input(^.className := "datepicker", ^.`type` := "text", ^.id := "add-datepicker", ^.disabled := p.progress, + ^.defaultValue := DateUtils.showDate(date)), + <.label(^.`for` := "add-datepicker", "Pobierz historię od") + ) + case StreamSource => EmptyVdom + }, + p.error.whenDefined { e => + <.p(^.className := "red-text", e) + }, + <.button(^.className := "btn waves-effect", ^.`type` := "submit", ^.disabled := p.progress, "Dodaj") + ) + ) + ) + } + + private val M = js.Dynamic.global.M + + def update = Callback { + val elem = dom.document.querySelector("#add-datepicker").asInstanceOf[HTMLInputElement] + if (elem != null) { + val date = DateUtils.parseDate(elem.value) + M.Datepicker.init(elem, js.Dynamic.literal( + format = "dd-mm-yyyy", + firstDay = 1, + i18n = DateUtils.i18n, + onClose = () => { + val date = DateUtils.parseDate(elem.value) + AppCircuit.dispatch(ObjectSourceChanged(HistorySource(date.getOrElse(new Date)))) + }, + minDate = new Date(Date.now() - 6 * 24 * 60 * 60 * 1000), + maxDate = new Date() + )) + val instance = M.Datepicker.getInstance(elem) + instance.setDate(date.getOrElse(new Date())) + } + M.updateTextFields() + } + } + + val component = ScalaComponent.builder[Props]("AddObjectView") + .stateless + .renderBackend[Backend] + .componentDidUpdate(_.backend.update) + .build + + def apply(props: Props) = component(props) +} diff --git a/front/src/main/scala/twitter/tweets/ObjectDeleteView.scala b/front/src/main/scala/twitter/tweets/ObjectDeleteView.scala new file mode 100644 index 0000000..064aad0 --- /dev/null +++ b/front/src/main/scala/twitter/tweets/ObjectDeleteView.scala @@ -0,0 +1,53 @@ +package twitter.tweets + +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import org.scalajs.dom + +import scala.scalajs.js + +object ObjectDeleteView { + + case class Props(query: String, + isDeleted: Boolean, + deleteCheck: Boolean, + onDeleteCheck: Boolean => Callback, + onSubmit: Callback) + + class Backend($: BackendScope[Props, Unit]) { + def render(p: Props): VdomElement = { + <.div(^.className := "modal", ^.id := "delete-modal", + <.div(^.className := "modal-content", + <.h4("Potwierdź usunięcie"), + <.p(s"Czy na pewno chcesz usunąć obiekt ", <.i(p.query), " z listy śledzonych obiektów?"), + <.p( + <.label( + <.input(^.`type` := "checkbox", ^.checked := p.deleteCheck, ^.onChange --> p.onDeleteCheck(!p.deleteCheck)), + <.span("Usuń permanentnie") + ) + ).when(!p.isDeleted) + ), + <.div(^.className := "modal-footer", + <.a(^.className := "modal-action modal-close waves-effect waves-green btn-flat", ^.href := "#!", + ^.onClick --> p.onSubmit, "Tak"), + <.a(^.className := "modal-action modal-close waves-effect waves-red btn-flat", ^.href := "#!", "Nie") + ) + ) + } + + private val M = js.Dynamic.global.M + + val start = Callback { + val elem = dom.document.querySelector(".modal") + M.Modal.init(elem) + } + } + + val component = ScalaComponent.builder[Props]("ObjectDeleteView") + .stateless + .renderBackend[Backend] + .componentDidMount(_.backend.start) + .build + + def apply(props: Props) = component(props) +} diff --git a/front/src/main/scala/twitter/tweets/ObjectView.scala b/front/src/main/scala/twitter/tweets/ObjectView.scala new file mode 100644 index 0000000..305b1dc --- /dev/null +++ b/front/src/main/scala/twitter/tweets/ObjectView.scala @@ -0,0 +1,150 @@ +package twitter.tweets + +import cats.Functor +import cats.implicits._ +import diode.Action +import diode.data.Pot +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import org.scalajs.dom +import org.scalajs.dom.raw.HTMLInputElement +import twitter.AppCircuit +import twitter.shared.{DateUtils, LoadingView, PaginationView} +import twitter.tweets.TweetsRest.ObjectStats + +import scala.scalajs.js +import scala.scalajs.js.Date + +object ObjectView { + + case class Props(objectStats: Pot[ObjectStats], + from: Option[Date], + to: Option[Date], + currentPage: Int, + onFromChange: Date => Callback, + onToChange: Date => Callback, + onPageChange: Int => Callback) + + class Backend($: BackendScope[Props, Unit]) { + def render(p: Props): VdomElement = { + <.div(^.className := "card", + <.div(^.className := "card-content", + if (p.objectStats.nonEmpty) { + val pageSize = 10 + val objectStats = p.objectStats.get + val days = objectStats.days.sortBy(-_.date.getTime()).slice(p.currentPage * pageSize, (p.currentPage + 1) * pageSize) + val tweetsCount = objectStats.days.foldLeft(0)((acc, x) => acc + x.count) + val showDate: Option[Date] => Option[String] = Functor[Option].lift(x => DateUtils.showDate(x)) + <.div( + <.h5(^.className := "title", s"Zarządzaj obiektem ", <.i(objectStats.query)), + <.div( + <.div(^.className := "input-field no-padding-left col s6", + <.input(^.className := "datepicker", ^.`type` := "text", ^.id := "stats-from", + ^.value := showDate(p.from).getOrElse(""), ^.disabled := p.objectStats.isPending), + <.label(^.`for` := "stats-from", "Od") + ), + <.div(^.className := "input-field no-padding-right col s6", + <.input(^.className := "datepicker", ^.`type` := "text", ^.id := "stats-to", + ^.value := showDate(p.to).getOrElse(""), ^.disabled := p.objectStats.isPending), + <.label(^.`for` := "stats-to", "Do") + ) + ), + <.p(s"Sumaryczna liczba tweetów: ${objectStats.allTweets}"), + <.p(s"Liczba tweetów w wybranym okresie: $tweetsCount"), + if (objectStats.days.nonEmpty) { + <.div( + <.table(^.className := "bordered", + <.thead( + <.tr( + <.th("Data"), + <.th("Liczba tweetów") + ) + ), + <.tbody( + days.toTagMod { d => + <.tr( + <.td(d.date.toDateString()), + <.td(d.count) + ) + } + ) + ), + PaginationView(PaginationView.Props( + itemsCount = objectStats.days.size, + currentPage = p.currentPage, + pageSize = pageSize, + onPageChange = page => p.onPageChange(page) + )) + ) + } else { + <.br + }, + <.a(^.className := "waves-effect waves-light btn red modal-trigger", ^.href := "#delete-modal", ^.disabled := p.objectStats.isPending, "Usuń obiekt") + ) + } else { + EmptyVdom + }, + if (p.objectStats.isPending) { + <.div(^.className := "center loading", + LoadingView() + ) + } else { + EmptyVdom + }, + p.objectStats.exceptionOption.whenDefined { e => + <.div(^.className := "center loading red-text", e.getMessage) + } + ) + ) + } + + private val M = js.Dynamic.global.M + + def update = Callback { + def findElem(selector: String): HTMLInputElement = { + dom.document.querySelector(selector).asInstanceOf[HTMLInputElement] + } + val findElemOpt: Option[String] => Option[HTMLInputElement] = Functor[Option].lift(x => findElem(x)) + def elemDate(selector: Option[String]): Option[Date] = { + findElemOpt(selector).flatMap(elem => DateUtils.parseDate(elem.value).toOption) + } + def updateDatepicker(elem: HTMLInputElement, action: Option[Date] => Action, minSelector: Option[String] = None, + maxSelector: Option[String] = None): Unit = { + if (elem != null) { + val min = elemDate(minSelector) + val max = elemDate(maxSelector) + M.Datepicker.init(elem, js.Dynamic.literal( + format = "dd-mm-yyyy", + firstDay = 1, + i18n = DateUtils.i18n, + showClearBtn = true, + onClose = () => { + val date = DateUtils.parseDate(elem.value) + AppCircuit.dispatch(action(date.toOption)) + }, + minDate = min.orNull, + maxDate = DateUtils.min(max.getOrElse(new Date), new Date()) + )) + + val date = DateUtils.parseDate(elem.value) + val instance = M.Datepicker.getInstance(elem) + instance.setDate(date.getOrElse(new Date())) + } + } + + val fromElem = findElem("#stats-from") + updateDatepicker(fromElem, x => ObjectStatsFromChanged(x), maxSelector = Some("#stats-to")) + val toElem = findElem("#stats-to") + updateDatepicker(toElem, x => ObjectStatsToChanged(x), minSelector = Some("#stats-from")) + M.updateTextFields() + } + } + + val component = ScalaComponent.builder[Props]("ObjectView") + .stateless + .renderBackend[Backend] + .componentDidUpdate(_.backend.update) + .build + + def apply(props: Props) = component(props) +} diff --git a/front/src/main/scala/twitter/tweets/TweetsListView.scala b/front/src/main/scala/twitter/tweets/TweetsListView.scala new file mode 100644 index 0000000..d110e84 --- /dev/null +++ b/front/src/main/scala/twitter/tweets/TweetsListView.scala @@ -0,0 +1,69 @@ +package twitter.tweets + +import diode.data.Pot +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import twitter.shared.{LoadingView, PaginationView} +import twitter.tweets.TweetsRest.TweetListItem + +object TweetsListView { + + case class Props(tweets: Pot[Seq[TweetListItem]], + selectedTweet: Option[Int], + currentPage: Int, + onTweetSelected: Int => Callback, + onPageChanged: Int => Callback) + + class Backend($: BackendScope[Props, Unit]) { + def render(p: Props): VdomElement = { + val pageSize = 10 + lazy val tweets = p.tweets.get.slice(p.currentPage * pageSize, (p.currentPage + 1) * pageSize) + <.div(^.className := "card", + <.div(^.className := "collection with-header", + <.div(^.className := "collection-header", + <.h5("Śledzone obiekty") + ), + if (p.tweets.nonEmpty) { + tweets.toTagMod { tweet => + val sourceIcon = if (tweet.source == "stream") "autorenew" else "history" + val typeIcon = if (tweet.objectType == "user") "person" else "description" + <.a(^.classSet1("collection-item", "active" -> p.selectedTweet.contains(tweet.id)), + ^.href := "#!", ^.onClick --> p.onTweetSelected(tweet.id), + if (tweet.deleted) <.span(^.className := "new badge red", vdomAttrVtString("data-badge-caption", ""), "Usunięto") else EmptyVdom, + <.i(^.className := "material-icons left", sourceIcon), + <.i(^.className := "material-icons left", typeIcon), + tweet.query + ) + } + } else if (p.tweets.isPending) { + <.div(^.className := "center loading", LoadingView()) + } else { + EmptyVdom + }, + p.tweets.exceptionOption.whenDefined { e => + <.div(^.className := "center loading red-text", e.getMessage) + }, + if (p.tweets.nonEmpty && tweets.nonEmpty) { + <.div( + PaginationView(PaginationView.Props( + itemsCount = p.tweets.get.size, + currentPage = p.currentPage, + pageSize = pageSize, + onPageChange = p.onPageChanged + )) + ) + } else { + EmptyVdom + } + ) + ) + } + } + + val component = ScalaComponent.builder[Props]("TweetsListView") + .stateless + .renderBackend[Backend] + .build + + def apply(props: Props) = component(props) +} diff --git a/front/src/main/scala/twitter/tweets/TweetsModel.scala b/front/src/main/scala/twitter/tweets/TweetsModel.scala new file mode 100644 index 0000000..c087221 --- /dev/null +++ b/front/src/main/scala/twitter/tweets/TweetsModel.scala @@ -0,0 +1,180 @@ +package twitter.tweets + +import diode.data.PotState._ +import diode.data.{Empty, Pot, PotAction} +import diode.{Action, ActionHandler, Effect, ModelRW} +import twitter.tweets.TweetsRest.{NewObjectInfo, ObjectStats, StreamSource, TweetListItem, TweetSource} + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.scalajs.js.Date + +case class TweetsModel(credentials: Option[String], + tweets: Pot[Seq[TweetListItem]], + newObjectInfo: NewObjectInfo, + newObjectProgress: Boolean, + newObjectError: Option[String], + selectedObjectId: Option[Int], + listCurrentPage: Int, + objectStats: Pot[ObjectStats], + objectStatsFrom: Option[Date], + objectStatsTo: Option[Date], + statsCurrentPage: Int, + objectDeleteAll: Boolean) + +object TweetsModel { + val initModel = TweetsModel( + credentials = None, + tweets = Pot.empty, + newObjectInfo = NewObjectInfo("", "keyword", StreamSource), + newObjectProgress = false, + newObjectError = None, + selectedObjectId = None, + listCurrentPage = 0, + objectStats = Pot.empty, + objectStatsFrom = None, + objectStatsTo = None, + statsCurrentPage = 0, + objectDeleteAll = false, + ) +} + +case class LoginSuccessful(credentials: String) extends Action +case object ClearCredentials extends Action + +case class UpdateTweetsList(potResult: Pot[Seq[TweetListItem]] = Empty) + extends PotAction[Seq[TweetListItem], UpdateTweetsList] { + override def next(newResult: Pot[Seq[TweetListItem]]) = UpdateTweetsList(newResult) +} +case class SelectObject(id: Int) extends Action +case class ListPageChanged(page: Int) extends Action + +case class ObjectQueryChanged(query: String) extends Action +case class ObjectTypeChanged(tweetType: String) extends Action +case class ObjectSourceChanged(tweetSource: TweetSource) extends Action +case object ObjectSubmit extends Action +case class NewObjectError(error: String) extends Action +case object NewObjectSuccessful extends Action + +case class UpdateObjectStats(objectId: Int, + from: Option[Date] = None, + to: Option[Date] = None, + potResult: Pot[ObjectStats] = Empty, + ) extends PotAction[ObjectStats, UpdateObjectStats] { + override def next(newResult: Pot[ObjectStats]): UpdateObjectStats = UpdateObjectStats(objectId, from, to, newResult) +} +case class ObjectStatsFromChanged(date: Option[Date]) extends Action +case class ObjectStatsToChanged(date: Option[Date]) extends Action +case class StatsPageChanged(page: Int) extends Action +case class ObjectDeleteAllChanged(all: Boolean) extends Action +case class ObjectDelete(objectId: Int) extends Action + +class TweetsHandler[M](modelRW: ModelRW[M, TweetsModel]) extends ActionHandler(modelRW) { + override def handle = { + case LoginSuccessful(credentials) => + val effect = Effect(Future.successful(UpdateTweetsList())) + updated(TweetsModel.initModel.copy(credentials = Some(credentials)), effect) + case ClearCredentials => + updated(value.copy(credentials = None)) + case action: UpdateTweetsList => + val effect = action.effect(TweetsRest.tweetsList(value.credentials.getOrElse("")))(identity) + action.handle { + case PotEmpty => + updated(value.copy(tweets = value.tweets.pending()), effect) + case PotPending => + noChange + case PotReady => + updated(value.copy(tweets = action.potResult)) + case PotUnavailable => + updated(value.copy(tweets = value.tweets.unavailable())) + case PotFailed => + val e = action.result.failed.get + updated(value.copy(tweets = value.tweets.fail(e))) + } + case SelectObject(id) => + value.selectedObjectId match { + case Some(i) if i == id => + updated(value.copy( + selectedObjectId = None, + objectStats = Pot.empty + )) + case _ => + val effect = Effect(Future.successful(UpdateObjectStats(id))) + updated(value.copy( + selectedObjectId = Some(id), + objectStatsFrom = None, + objectStatsTo = None, + objectDeleteAll = false + ), effect) + } + case ListPageChanged(p) => + updated(value.copy(listCurrentPage = p)) + case ObjectQueryChanged(v) => + updated(value.copy(newObjectInfo = value.newObjectInfo.copy(query = v))) + case ObjectTypeChanged(v) => + updated(value.copy(newObjectInfo = value.newObjectInfo.copy(objectType = v))) + case ObjectSourceChanged(v) => + updated(value.copy(newObjectInfo = value.newObjectInfo.copy(source = v))) + case ObjectSubmit => + val effect = Effect(TweetsRest.newObject(value.credentials.getOrElse(""), value.newObjectInfo) + .map { + case Left(e) => NewObjectError(e.getMessage) + case Right(_) => NewObjectSuccessful + }) + updated(value.copy(newObjectProgress = true), effect) + case NewObjectError(e) => + updated(value.copy(newObjectProgress = false, newObjectError = Some(e))) + case NewObjectSuccessful => + val effect = Effect(Future.successful(UpdateTweetsList())) + updated(value.copy( + newObjectInfo = TweetsModel.initModel.newObjectInfo, + newObjectProgress = false, + newObjectError = None + ), effect) + case action: UpdateObjectStats => + val effect = action.effect( + TweetsRest.objectStats(value.credentials.getOrElse(""), action.objectId, action.from, action.to) + )(identity) + action.handle { + case PotEmpty => + updated(value.copy(objectStats = value.objectStats.pending()), effect) + case PotPending => + noChange + case PotReady => + if (value.selectedObjectId.contains(action.potResult.get.id)) { + updated(value.copy(objectStats = action.potResult)) + } else { + noChange + } + case PotUnavailable => + updated(value.copy(objectStats = value.objectStats.unavailable())) + case PotFailed => + val e = action.result.failed.get + updated(value.copy(objectStats = value.objectStats.fail(e))) + } + case ObjectStatsFromChanged(date) => + val effect = Effect(Future.successful(UpdateObjectStats( + objectId = value.selectedObjectId.getOrElse(0), + from = date, + to = value.objectStatsTo + ))) + updated(value.copy(objectStatsFrom = date), effect) + case ObjectStatsToChanged(date) => + val effect = Effect(Future.successful(UpdateObjectStats( + objectId = value.selectedObjectId.getOrElse(0), + from = value.objectStatsFrom, + to = date + ))) + updated(value.copy(objectStatsTo = date), effect) + case StatsPageChanged(p) => + updated(value.copy(statsCurrentPage = p)) + case ObjectDeleteAllChanged(all) => + updated(value.copy(objectDeleteAll = all)) + case ObjectDelete(id) => + val isDeleted = value.objectStats.toOption.exists(_.deleted) + val effect = Effect(Future.successful(SelectObject(id))) >> + Effect(TweetsRest.deleteObject(value.credentials.getOrElse(""), id, value.objectDeleteAll || isDeleted) + .map(_ => UpdateTweetsList())) + updated(value, effect) + } +} diff --git a/front/src/main/scala/twitter/tweets/TweetsRest.scala b/front/src/main/scala/twitter/tweets/TweetsRest.scala new file mode 100644 index 0000000..246da9b --- /dev/null +++ b/front/src/main/scala/twitter/tweets/TweetsRest.scala @@ -0,0 +1,114 @@ +package twitter.tweets + +import cats.implicits._ +import io.circe.generic.auto._ +import io.circe.parser.decode +import io.circe.syntax._ +import io.circe.{Decoder, Encoder, HCursor, Json} +import org.scalajs.dom.ext.Ajax +import org.scalajs.dom.ext.Ajax.InputData +import twitter.MainApp +import twitter.login.LoginRest.{AjaxFuture, MessageInfo, authorizationHeader} + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.scalajs.js.Date + +object TweetsRest { + + case class TweetListItem(id: Int, source: String, objectType: String, query: String, deleted: Boolean) + + def tweetsList(credentials: String): Future[Seq[TweetListItem]] = { + val url = s"${MainApp.apiUrl}/objects" + val headers = authorizationHeader(credentials) + Ajax.get(url, headers = headers) + .map(r => decode[Seq[TweetListItem]](r.responseText)) + .recoverAjax() + .flatMap { + case Right(xs) => Future.successful(xs) + case Left(e) => Future.failed(e) + } + } + + sealed trait TweetSource + case object StreamSource extends TweetSource + case class HistorySource(history: Date) extends TweetSource + + case class NewObjectInfo(query: String, objectType: String, source: TweetSource) + + object NewObjectInfo { + val userType = "user" + val keywordType = "keyword" + } + + implicit val encodeNewObjectInfo: Encoder[NewObjectInfo] = + (a: NewObjectInfo) => { + a.source match { + case StreamSource => + Json.obj( + ("source", Json.fromString("stream")), + ("objectType", Json.fromString(a.objectType)), + ("query", Json.fromString(a.query)) + ) + case HistorySource(history) => + Json.obj( + ("source", Json.fromString("history")), + ("objectType", Json.fromString(a.objectType)), + ("history", Json.fromString(history.toUTCString())), + ("query", Json.fromString(a.query)) + ) + } + } + + def newObject(credentials: String, tweet: NewObjectInfo): Future[Either[Exception, MessageInfo]] = { + val url = s"${MainApp.apiUrl}/objects" + val headers = authorizationHeader(credentials) + val json = tweet.asJson.noSpaces + Ajax.post(url, headers = headers, data = InputData.str2ajax(json)) + .map(r => decode[MessageInfo](r.responseText)) + .recoverAjax() + } + + case class DayStats(date: Date, count: Int) + + case class ObjectStats(id: Int, query: String, deleted: Boolean, allTweets: Int, days: Seq[DayStats]) + + implicit val decodeDayStats: Decoder[DayStats] = + (c: HCursor) => for { + date <- c.downField("date").as[String] + count <- c.downField("count").as[Int] + } yield DayStats(new Date(Date.parse(date)), count) + + def addParamsToUrl(url: String, params: List[(String, String)], first: Boolean = true): String = params match { + case Nil => url + case (p, v) :: t => addParamsToUrl(s"$url${if (first) "?" else "&"}$p=$v", t, first = false) + } + + def objectStats(credentials: String, objectId: Int, from: Option[Date] = None, to: Option[Date] = None): Future[ObjectStats] = { + val params = List( + from.map(x => ("from", x.toUTCString())), + to.map(x => ("to", new Date(x.getTime() + 24 * 60 * 60 * 1000).toUTCString())) + ).flatten + val url = addParamsToUrl(s"${MainApp.apiUrl}/objects/$objectId", params) + val headers = authorizationHeader(credentials) + Ajax.get(url, headers = headers) + .map(r => decode[ObjectStats](r.responseText)) + .recoverAjax() + .flatMap { + case Right(xs) => Future.successful(xs) + case Left(e) => Future.failed(e) + } + } + + def deleteObject(credentials: String, objectId: Int, all: Boolean): Future[MessageInfo] = { + val url = addParamsToUrl(s"${MainApp.apiUrl}/objects/$objectId", List(("all", all.show))) + val headers = authorizationHeader(credentials) + Ajax.delete(url, headers = headers) + .map(r => decode[MessageInfo](r.responseText)) + .recoverAjax() + .flatMap { + case Right(xs) => Future.successful(xs) + case Left(e) => Future.failed(e) + } + } +} diff --git a/front/src/main/scala/twitter/tweets/TweetsView.scala b/front/src/main/scala/twitter/tweets/TweetsView.scala new file mode 100644 index 0000000..ad97550 --- /dev/null +++ b/front/src/main/scala/twitter/tweets/TweetsView.scala @@ -0,0 +1,67 @@ +package twitter.tweets + +import diode.Action +import diode.data.PotState.PotEmpty +import japgolly.scalajs.react._ +import japgolly.scalajs.react.vdom.html_<^._ +import twitter.shared.LoadingView + +object TweetsView { + + case class Props(tweetsModel: TweetsModel, dispatch: Action => Callback) + + class Backend($: BackendScope[Props, Unit]) { + def render(p: Props): VdomElement = { + <.div(^.className := "row", + <.div(^.className := "col s6", + AddObjectView(AddObjectView.Props( + info = p.tweetsModel.newObjectInfo, + progress = p.tweetsModel.newObjectProgress, + error = p.tweetsModel.newObjectError, + onQueryChange = x => p.dispatch(ObjectQueryChanged(x)), + onTypeChange = x => p.dispatch(ObjectTypeChanged(x)), + onSourceChange = x => p.dispatch(ObjectSourceChanged(x)), + onSubmit = p.dispatch(ObjectSubmit) + )), + TweetsListView(TweetsListView.Props( + tweets = p.tweetsModel.tweets, + selectedTweet = p.tweetsModel.selectedObjectId, + currentPage = p.tweetsModel.listCurrentPage, + onTweetSelected = id => p.dispatch(SelectObject(id)), + onPageChanged = page => p.dispatch(ListPageChanged(page)) + )) + ), + <.div(^.className := "col s6", + if (p.tweetsModel.objectStats.state == PotEmpty) { + EmptyVdom + } else { + ObjectView(ObjectView.Props( + objectStats = p.tweetsModel.objectStats, + from = p.tweetsModel.objectStatsFrom, + to = p.tweetsModel.objectStatsTo, + currentPage = p.tweetsModel.statsCurrentPage, + onFromChange = _ => Callback.empty, + onToChange = _ => Callback.empty, + onPageChange = page => p.dispatch(StatsPageChanged(page)) + )) + } + ), + ObjectDeleteView(ObjectDeleteView.Props( + query = p.tweetsModel.objectStats.toOption.map(_.query).getOrElse(""), + isDeleted = p.tweetsModel.objectStats.toOption.exists(_.deleted), + deleteCheck = p.tweetsModel.objectDeleteAll, + onDeleteCheck = x => p.dispatch(ObjectDeleteAllChanged(x)), + onSubmit = p.dispatch(ObjectDelete(p.tweetsModel.selectedObjectId.getOrElse(0))) + )) + ) + } + } + + val component = ScalaComponent.builder[Props]("TweetsView") + .stateless + .renderBackend[Backend] + .build + + def apply(tweetsModel: TweetsModel, dispatch: Action => Callback) = + component(Props(tweetsModel, dispatch)) +} diff --git a/misc/screenshots/screen1.png b/misc/screenshots/screen1.png new file mode 100644 index 0000000..791be53 Binary files /dev/null and b/misc/screenshots/screen1.png differ diff --git a/twitter-saver/config.go b/twitter-saver/config.go new file mode 100644 index 0000000..d99ad73 --- /dev/null +++ b/twitter-saver/config.go @@ -0,0 +1,30 @@ +package main + +import ( + "io/ioutil" + + "gitlab.com/kompu/twitter-saver/core" + "gopkg.in/yaml.v2" +) + +type Config struct { + Db core.DbConfig + Twitter core.TwitterConfig + Json core.JsonConfig + AutoDeleteDays *int `yaml:"autoDeleteDays"` +} + +func readConfig(path *string) (*Config, error) { + data, err := ioutil.ReadFile(*path) + if err != nil { + return nil, err + } + + config := Config{} + err = yaml.Unmarshal(data, &config) + if err != nil { + return nil, err + } + + return &config, nil +} diff --git a/twitter-saver/config.yaml b/twitter-saver/config.yaml new file mode 100644 index 0000000..81b9df2 --- /dev/null +++ b/twitter-saver/config.yaml @@ -0,0 +1,19 @@ +db: + host: + port: + user: + password: + dbName: + sslMode: disable +twitter: + consumerKey: + consumerSecret: + token: + tokenSecret: +json: + all: false + fields: + - truncated + - user.name + - user.entities +autoDeleteDays: null diff --git a/twitter-saver/history.go b/twitter-saver/history.go new file mode 100644 index 0000000..f4dce70 --- /dev/null +++ b/twitter-saver/history.go @@ -0,0 +1,100 @@ +package main + +import ( + "log" + "sync" + "time" + + "github.com/dghubble/go-twitter/twitter" + "github.com/jinzhu/gorm" + "gitlab.com/kompu/twitter-saver/core" +) + +type SafeFlag struct { + v bool + mux sync.Mutex +} + +func (f *SafeFlag) SetFlag(v bool) { + f.mux.Lock() + f.v = v + f.mux.Unlock() +} + +func (f *SafeFlag) CheckFlag() bool { + f.mux.Lock() + defer f.mux.Unlock() + return f.v +} + +func startDownloadingHistory(config *Config, db *gorm.DB, client *twitter.Client, downloading *SafeFlag) { + if downloading.CheckFlag() { + log.Println("Already downloading history.") + return + } + + downloading.SetFlag(true) + if config.AutoDeleteDays != nil { + log.Println("Removing old tweets.") + days := time.Duration(*config.AutoDeleteDays) + date := time.Now().Add(days * -24 * time.Hour) + err := core.RemoveTweetsOlderThan(db, date) + if err != nil { + log.Println(err) + } + } + + log.Println("Started downloading history objects.") + + objects := core.FindHistoryObjects(db) + for _, object := range objects { + var date *time.Time + var sinceId int64 + if object.HistoryFrom != nil && !object.HistoryDone { + date = object.HistoryFrom + } else { + var err error + sinceId, err = core.LatestTweetForObject(db, object.ID) + if err != nil { + log.Println("Error getting from object", object.ID, "latest sinceId:", err) + } + } + + var query string + if object.Type == core.UserType { + query = "from:" + object.Query + } else { + query = object.Query + } + log.Printf("Downloading history for '%s'", query) + + lastMaxId := int64(0) + for { + results, maxId, err := searchSince(client, query, date, sinceId, lastMaxId) + for _, tweet := range results { + err := core.InsertTweet(db, &tweet, object.ID, config.Json.Fields, config.Json.All) + if err != nil { + log.Println("Json storing error for history object", query, ":", err) + } + } + + if err != nil { + log.Println("Error downloading history for", query, ":", err) + } else if object.HistoryFrom != nil && !object.HistoryDone { + core.UpdateObjectHistory(db, &object, true) + } + + if maxId != 0 { + log.Println("Limit exceeded for history object", query) + time.Sleep(15 * time.Minute) + log.Println("Resuming downloading", query) + lastMaxId = maxId + } else { + break + } + } + } + + log.Println("Finished downloading history objects.") + downloading.SetFlag(false) +} diff --git a/twitter-saver/main.go b/twitter-saver/main.go new file mode 100644 index 0000000..b638855 --- /dev/null +++ b/twitter-saver/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "flag" + "log" + "os" + "os/signal" + "syscall" + + "github.com/robfig/cron" + "gitlab.com/kompu/twitter-saver/core" +) + +func main() { + configPath := flag.String("config", "config.yaml", "Path to config file") + flag.Parse() + + conf, err := readConfig(configPath) + if err != nil { + log.Fatalln(err) + } + + conn, err := conf.Db.ConnectionString() + if err != nil { + log.Fatalln(err) + } + db := core.Connect(conn) + defer db.Close() + + client, err := conf.Twitter.TwitterClient() + if err != nil { + log.Fatalln(err) + } + + downloading := SafeFlag{v: false} + + scheduler := cron.New() + scheduler.AddFunc("0 0 0 * * *", func() { + startDownloadingHistory(conf, db, client, &downloading) + }) + scheduler.Start() + + go startStreaming(conf, db, client) + go startDownloadingHistory(conf, db, client, &downloading) + + exitSignal := make(chan os.Signal) + signal.Notify(exitSignal, syscall.SIGINT, syscall.SIGTERM) + <-exitSignal + + scheduler.Stop() +} diff --git a/twitter-saver/search.go b/twitter-saver/search.go new file mode 100644 index 0000000..2f953e1 --- /dev/null +++ b/twitter-saver/search.go @@ -0,0 +1,60 @@ +package main + +import ( + "time" + + "github.com/dghubble/go-twitter/twitter" +) + +func min(a, b int64) int64 { + if a < b { + return a + } + return b +} + +func limitExceeded(apiError *twitter.APIError) bool { + for _, err := range apiError.Errors { + if err.Code == 88 { + return true + } + } + return false +} + +func searchSince(client *twitter.Client, query string, date *time.Time, sinceId int64, maxId int64) ([]twitter.Tweet, int64, error) { + results := make([]twitter.Tweet, 0) + + for { + search, _, err := client.Search.Tweets(&twitter.SearchTweetParams{ + Query: query, + Count: 100, + MaxID: maxId, + SinceID: sinceId, + }) + if apiErr, ok := err.(twitter.APIError); ok && limitExceeded(&apiErr) { + return results, maxId, nil + } + if err != nil { + return results, 0, err + } + if len(search.Statuses) == 0 { + return results, 0, nil + } + + if maxId == 0 { + maxId = search.Statuses[0].ID - 1 + } + for _, tweet := range search.Statuses { + maxId = min(maxId, tweet.ID-1) + t, err := time.Parse(time.RubyDate, tweet.CreatedAt) + if err != nil { + return results, 0, err + } + if date != nil && t.Before(*date) { + return results, 0, nil + } + results = append(results, tweet) + } + } +} diff --git a/twitter-saver/stream.go b/twitter-saver/stream.go new file mode 100644 index 0000000..49a5c12 --- /dev/null +++ b/twitter-saver/stream.go @@ -0,0 +1,147 @@ +package main + +import ( + "log" + "time" + + "github.com/dghubble/go-twitter/twitter" + "github.com/jinzhu/gorm" + "gitlab.com/kompu/twitter-saver/core" +) + +type StreamInfo struct { + ID uint + Type core.ObjectType + Query string +} + +type Streams map[StreamInfo]chan int + +func streamsToRemove(running Streams, newObjects []core.Object) []StreamInfo { + newInfo := make(map[StreamInfo]bool) + for _, v := range newObjects { + newInfo[StreamInfo{ID: v.ID, Type: v.Type, Query: v.Query}] = true + } + + rem := make([]StreamInfo, 0) + for k := range running { + _, ok := newInfo[k] + if !ok { + rem = append(rem, k) + } + } + return rem +} + +func streamsToAdd(running Streams, newObjects []core.Object) []StreamInfo { + add := make([]StreamInfo, 0) + for _, v := range newObjects { + s := StreamInfo{ID: v.ID, Type: v.Type, Query: v.Query} + _, ok := running[s] + if !ok { + add = append(add, s) + } + } + return add +} + +func startStream(config *Config, db *gorm.DB, client *twitter.Client, info StreamInfo, + quit <-chan int, errors chan<- StreamInfo) { + log.Println("Starting stream", info) + + var query string + if info.Type == core.UserType { + query = "from:" + info.Query + } else { + query = info.Query + } + params := &twitter.StreamFilterParams{ + Track: []string{query}, + StallWarnings: twitter.Bool(true), + } + + stream, err := client.Streams.Filter(params) + if err != nil { + log.Println("Stream error:", err) + errors <- info + return + } + + demux := twitter.NewSwitchDemux() + demux.Tweet = func(tweet *twitter.Tweet) { + err := core.InsertTweet(db, tweet, info.ID, config.Json.Fields, config.Json.All) + if err != nil { + log.Println("Json storing error for stream", info, ":", err) + } + } + demux.Warning = func(warning *twitter.StallWarning) { + log.Println("Stream", info, "is falling behind. Warning code:", + warning.Code, ", message: ", warning.Message) + } + demux.StreamLimit = func(limit *twitter.StreamLimit) { + log.Println("Stream", info, "reached limit,", + limit.Track, "undelivered matches.") + errors <- info + stream.Stop() + return + } + demux.StreamDisconnect = func(disconnect *twitter.StreamDisconnect) { + log.Println("Stream", info, "disconnected. Disconection code:", + disconnect.Code, ", reason:", disconnect.Reason) + errors <- info + stream.Stop() + return + } + + for { + select { + case message := <-stream.Messages: + demux.Handle(message) + case <-quit: + log.Println("Stoping stream", info) + stream.Stop() + return + } + } +} + +func startStreaming(config *Config, db *gorm.DB, client *twitter.Client) { + running := make(Streams) + errors := make(chan StreamInfo) + ticker := time.NewTicker(60 * time.Second) + + work := func() { + streams := core.FindStreamObjects(db) + rem := streamsToRemove(running, streams) + add := streamsToAdd(running, streams) + + if len(add) > 0 { + log.Println("Streams to add:", add) + } + if len(rem) > 0 { + log.Println("Streams to remove:", rem) + } + + for _, v := range rem { + quit := running[v] + quit <- 0 + delete(running, v) + } + + for _, v := range add { + quit := make(chan int) + running[v] = quit + go startStream(config, db, client, v, quit, errors) + } + } + work() + + for { + select { + case v := <-errors: + delete(running, v) + case <-ticker.C: + work() + } + } +} diff --git a/web/config.go b/web/config.go new file mode 100644 index 0000000..4e64fbf --- /dev/null +++ b/web/config.go @@ -0,0 +1,28 @@ +package main + +import ( + "io/ioutil" + + "gitlab.com/kompu/twitter-saver/core" + "gopkg.in/yaml.v2" +) + +type Config struct { + Db core.DbConfig + Web core.WebConfig +} + +func readConfig(path *string) (*Config, error) { + data, err := ioutil.ReadFile(*path) + if err != nil { + return nil, err + } + + config := Config{} + err = yaml.Unmarshal(data, &config) + if err != nil { + return nil, err + } + + return &config, nil +} diff --git a/web/config.yaml b/web/config.yaml new file mode 100644 index 0000000..d479a54 --- /dev/null +++ b/web/config.yaml @@ -0,0 +1,15 @@ +db: + host: + port: + user: + password: + dbName: + sslMode: disable +web: + port: 8080 + secret: + users: + - username: admin + password: admin + - username: test + password: test diff --git a/web/main.go b/web/main.go new file mode 100644 index 0000000..31564eb --- /dev/null +++ b/web/main.go @@ -0,0 +1,130 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "strconv" + "time" + + jwt "github.com/appleboy/gin-jwt" + "github.com/gin-gonic/gin" + "gitlab.com/kompu/twitter-saver/core" +) + +func main() { + configPath := flag.String("config", "config.yaml", "Path to config file") + flag.Parse() + + conf, err := readConfig(configPath) + if err != nil { + log.Fatalln(err) + } + if conf.Web.Port == nil { + log.Fatalln("Web server port not specified.") + } + if conf.Web.Secret == nil { + log.Fatalln("Web secret key not specified.") + } + + accounts, err := conf.Web.WebAccounts() + if err != nil { + log.Fatalln(err) + } + + conn, err := conf.Db.ConnectionString() + if err != nil { + log.Fatalln(err) + } + db := core.Connect(conn) + defer db.Close() + + r := gin.Default() + + r.Static("/assets", "./public") + r.StaticFile("/", "./public/index.html") + + authMiddleware := jwt.GinJWTMiddleware{ + Realm: "twitter-saver", + Key: []byte(*conf.Web.Secret), + Timeout: 24 * time.Hour, + MaxRefresh: 7 * 24 * time.Hour, + Authenticator: func(userId string, password string, c *gin.Context) (string, bool) { + if pass, ok := accounts[userId]; ok { + return userId, pass == password + } + return userId, false + }, + } + + authorized := r.Group("/api") + + authorized.POST("/login", authMiddleware.LoginHandler) + + authorized.Use(authMiddleware.MiddlewareFunc()) + + authorized.GET("/refresh_token", authMiddleware.RefreshHandler) + + authorized.GET("/objects", func(c *gin.Context) { + objects := listObjects(db) + c.JSON(http.StatusOK, objects) + }) + authorized.POST("/objects", func(c *gin.Context) { + var json ObjectAdd + if err := c.ShouldBindJSON(&json); err == nil { + err := addObject(db, &json) + if err != nil { + if err.Error() == "Two streams already exist." { + c.JSON(http.StatusNotAcceptable, gin.H{"error": err.Error()}) + } else { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + } + } else { + c.JSON(http.StatusOK, gin.H{"message": "ok"}) + } + } else { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + } + }) + + authorized.GET("/objects/:id", func(c *gin.Context) { + idStr := c.Param("id") + id, err := strconv.Atoi(idStr) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Wrong 'id' format"}) + return + } + from := c.DefaultQuery("from", "") + to := c.DefaultQuery("to", "") + stats, err := objectStats(db, id, from, to) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, stats) + }) + authorized.DELETE("/objects/:id", func(c *gin.Context) { + idStr := c.Param("id") + allStr := c.DefaultQuery("all", "false") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Wrong 'id' format"}) + return + } + all, err := strconv.ParseBool(allStr) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Wrong 'all' format"}) + return + } + err = deleteObject(db, uint(id), all) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + } else { + c.JSON(http.StatusOK, gin.H{"message": "ok"}) + } + }) + + addr := fmt.Sprintf(":%d", *conf.Web.Port) + r.Run(addr) +} diff --git a/web/objects.go b/web/objects.go new file mode 100644 index 0000000..95c498e --- /dev/null +++ b/web/objects.go @@ -0,0 +1,254 @@ +package main + +import ( + "database/sql" + "errors" + "fmt" + "sort" + "time" + + "github.com/jinzhu/gorm" + "gitlab.com/kompu/twitter-saver/core" +) + +type ObjectList struct { + ID uint `json:"id"` + Source string `json:"source"` + Type string `json:"objectType"` + Query string `json:"query"` + Deleted bool `json:"deleted"` +} + +func objectSource(source core.ObjectSource) string { + switch source { + case core.HistorySource: + return "history" + case core.StreamSource: + return "stream" + default: + return "unknown" + } +} + +func toObjectSource(str string) (core.ObjectSource, error) { + switch str { + case "history": + return core.HistorySource, nil + case "stream": + return core.StreamSource, nil + default: + return 0, errors.New("Unknown source " + str) + } +} + +func objectType(objectType core.ObjectType) string { + switch objectType { + case core.UserType: + return "user" + case core.KeywordType: + return "keyword" + default: + return "unknown" + } +} + +func toObjectType(str string) (core.ObjectType, error) { + switch str { + case "user": + return core.UserType, nil + case "keyword": + return core.KeywordType, nil + default: + return 0, errors.New("Unknown type " + str) + } +} + +type byCreatedAt []core.Object + +func (s byCreatedAt) Len() int { + return len(s) +} +func (s byCreatedAt) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} +func (s byCreatedAt) Less(i, j int) bool { + return s[i].CreatedAt.After(s[j].CreatedAt) +} + +func listObjects(db *gorm.DB) []ObjectList { + objects := core.FindAllObjects(db) + sort.Sort(byCreatedAt(objects)) + + res := make([]ObjectList, len(objects)) + for i, v := range objects { + res[i] = ObjectList{ + ID: v.ID, + Source: objectSource(v.Source), + Type: objectType(v.Type), + Query: v.Query, + Deleted: v.DeletedAt != nil, + } + } + return res +} + +type ObjectAdd struct { + Source string `json:"source" binding:"required"` + Type string `json:"objectType" binding:"required"` + Query string `json:"query" binding:"required"` + History string `json:"history"` +} + +func addObject(db *gorm.DB, object *ObjectAdd) error { + source, err := toObjectSource(object.Source) + if err != nil { + return err + } + + typ, err := toObjectType(object.Type) + if err != nil { + return err + } + + if source == core.StreamSource { + count := core.CountStreamObjects(db) + if count >= 2 { + return errors.New("Two streams already exist.") + } + } + + if object.History != "" { + history, err := time.Parse(time.RFC1123, object.History) + if err != nil { + return err + } + db.Create(&core.Object{ + Source: source, + Type: typ, + Query: object.Query, + HistoryFrom: &history, + }) + } else { + db.Create(&core.Object{ + Source: source, + Type: typ, + Query: object.Query, + }) + } + + return nil +} + +func deleteObject(db *gorm.DB, id uint, all bool) error { + var object core.Object + db.Unscoped().First(&object, id) + if object.ID == id { + if all { + db.Unscoped().Delete(&object) + } else { + db.Delete(&object) + } + return nil + } else { + return errors.New(fmt.Sprintf("Object with id=%d doesn't exist.", id)) + } +} + +type DayStats struct { + Date string `json:"date"` + Count int `json:"count"` +} + +type ObjectStats struct { + ID int `json:"id"` + Query string `json:"query"` + Deleted bool `json:"deleted"` + AllTweets int `json:"allTweets"` + Days []DayStats `json:"days"` +} + +func objectStats(db *gorm.DB, id int, from string, to string) (*ObjectStats, error) { + var obj core.Object + db.Unscoped().First(&obj, id) + + var allCount int + db.Model(&core.Tweet{}).Where("object_id = ?", id).Count(&allCount) + + var rows *sql.Rows + var err error + + if from != "" && to != "" { + fromTime, err := time.Parse(time.RFC1123, from) + if err != nil { + return nil, err + } + toTime, err := time.Parse(time.RFC1123, to) + if err != nil { + return nil, err + } + rows, err = db.Raw(`SELECT (published_at AT TIME ZONE 'Europe/Warsaw')::date, COUNT(*) + FROM tweets + WHERE object_id = ? AND published_at >= ? AND published_at <= ? + GROUP BY 1 ORDER BY 1`, id, fromTime, toTime).Rows() + if err != nil { + return nil, err + } + } else if from != "" { + fromTime, err := time.Parse(time.RFC1123, from) + if err != nil { + return nil, err + } + rows, err = db.Raw( + `SELECT (published_at AT TIME ZONE 'Europe/Warsaw')::date, COUNT(*) + FROM tweets + WHERE object_id = ? AND published_at >= ? + GROUP BY 1 ORDER BY 1`, id, fromTime).Rows() + if err != nil { + return nil, err + } + } else if to != "" { + toTime, err := time.Parse(time.RFC1123, to) + if err != nil { + return nil, err + } + rows, err = db.Raw( + `SELECT (published_at AT TIME ZONE 'Europe/Warsaw')::date, COUNT(*) + FROM tweets + WHERE object_id = ? AND published_at <= ? + GROUP BY 1 ORDER BY 1`, id, toTime).Rows() + if err != nil { + return nil, err + } + } else { + rows, err = db.Raw( + `SELECT (published_at AT TIME ZONE 'Europe/Warsaw')::date, COUNT(*) + FROM tweets + WHERE object_id = ? + GROUP BY 1 ORDER BY 1`, id).Rows() + if err != nil { + return nil, err + } + } + + days := make([]DayStats, 0) + for rows.Next() { + var createdAt time.Time + var count int + err := rows.Scan(&createdAt, &count) + if err != nil { + return nil, err + } + days = append(days, DayStats{ + Date: createdAt.Format(time.RFC1123), + Count: count, + }) + } + + return &ObjectStats{ + ID: id, + Query: obj.Query, + Deleted: obj.DeletedAt != nil, + AllTweets: allCount, + Days: days, + }, nil +}