initial commit
This commit is contained in:
shim_
@@ -0,0 +1,84 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import cats.data.Xor
|
||||
import monix.async.Task
|
||||
import monix.execution.{Cancelable, Scheduler}
|
||||
import org.scalajs._
|
||||
|
||||
|
||||
import scala.concurrent.{ExecutionContext}
|
||||
import scala.scalajs.js
|
||||
import scala.scalajs.js.{Any, Array, JSON}
|
||||
|
||||
/**
|
||||
* Created by marvin on 3/26/16.
|
||||
*/
|
||||
|
||||
private[api] abstract class DataSourceCompanion{
|
||||
val Default: DataSource = new DataSource {
|
||||
|
||||
def request(url: URL,method: String = "GET",data: Option[String] = None,headers: Seq[(String,String)] = Seq.empty,auth: Option[(String,String)] = None) = Task.create[Xor[dom.ErrorEvent,(Int,String)]]{ (callback,_) =>
|
||||
val xhr = new dom.XMLHttpRequest()
|
||||
import js.JSConverters._
|
||||
val open = xhr.open(method,url.toString,true,_: String,_: String)
|
||||
auth match {
|
||||
case Some((user,pass)) => open(user,pass)
|
||||
case _ => open(null,null)
|
||||
}
|
||||
headers.foreach{
|
||||
case (k,v) => xhr.setRequestHeader(k,v)
|
||||
}
|
||||
xhr.onload = { (e: dom.Event) =>
|
||||
callback.onSuccess(Xor.Right((xhr.status,xhr.responseText)))
|
||||
}
|
||||
xhr.onerror = { (e: dom.ErrorEvent) =>
|
||||
callback.onSuccess(Xor.Left(e))
|
||||
}
|
||||
data.fold(xhr.send())(data => xhr.send(data))
|
||||
new Cancelable {
|
||||
override def cancel(): Unit = xhr.abort()
|
||||
}
|
||||
}
|
||||
|
||||
override implicit def executionContext: ExecutionContext = ExecutionContext.Implicits.global
|
||||
|
||||
|
||||
override def httpRequest(url: URL, method: String, data: Option[String],headers: Seq[(String,String)] = Seq.empty, auth: Option[(String, String)]): Task[(Int, String)] = request(url,method,data,headers,auth).flatMap{
|
||||
case Xor.Right((status: Int,text: String)) =>
|
||||
Task.now((status,text))
|
||||
case Xor.Left(err) => Task.error(new Exception("xhr request failed"))
|
||||
}
|
||||
|
||||
override implicit def scheduler: Scheduler = Scheduler.trampoline()
|
||||
|
||||
//JsonP
|
||||
override def json(url: URL, auth: Option[(String, String)],callbackParameter: String = "jsonp"): Task[(Int, String)] = Task.create{ (reg,scheduler: Scheduler) =>
|
||||
val tag = dom.document.createElement("script")
|
||||
tag.setAttribute("async","true")
|
||||
tag.setAttribute("type","text/javascript")
|
||||
val functionName = s"jsonp_${scala.util.Random.alphanumeric.take(6).mkString}"
|
||||
val window = dom.window.asInstanceOf[js.Dynamic]
|
||||
window.updateDynamic(functionName)(((json: js.Object) => {
|
||||
window.updateDynamic(functionName)(null)
|
||||
tag.parentNode.removeChild(tag)
|
||||
val text: String = JSON.stringify(json)
|
||||
reg.onSuccess((200,text))
|
||||
}): js.Function1[js.Object,Unit])
|
||||
scheduler.scheduleOnce(30,TimeUnit.SECONDS,new Runnable {
|
||||
override def run(): Unit = if(window.selectDynamic(functionName) != null){
|
||||
window.updateDynamic(functionName)(null)
|
||||
tag.parentNode.removeChild(tag)
|
||||
reg.onError(new Exception("JsonP request failed!"))
|
||||
}
|
||||
})
|
||||
tag.setAttribute("src",s"$url${if(url.toString.contains("?")) "&" else "?"}$callbackParameter=$functionName")
|
||||
dom.document.body.appendChild(tag)
|
||||
new Cancelable {
|
||||
override def cancel(): Unit = ()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
/**
|
||||
* Created by marvin on 3/26/16.
|
||||
*/
|
||||
|
||||
|
||||
object JsonParser {
|
||||
|
||||
//def parse(json: String) = io.circe.parser.parse(json)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import org.scalajs.dom
|
||||
|
||||
import scala.scalajs.js.URIUtils
|
||||
|
||||
/**
|
||||
* Created by marvin on 3/27/16.
|
||||
*/
|
||||
object UrlEncoder {
|
||||
|
||||
def encode(s: String,enc: String) = URIUtils.encodeURIComponent(s)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import java.net.HttpURLConnection
|
||||
import java.util.Base64
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
import monix.async.Task
|
||||
import monix.execution.{Cancelable, Scheduler}
|
||||
|
||||
import scala.concurrent.ExecutionContext
|
||||
import scala.io.Source
|
||||
|
||||
/**
|
||||
* Created by marvin on 3/26/16.
|
||||
*/
|
||||
private[api] abstract class DataSourceCompanion{
|
||||
val Default: DataSource = new DataSource {
|
||||
|
||||
override implicit def executionContext: ExecutionContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(4))
|
||||
|
||||
override def httpRequest(url: URL, method: String, data: Option[String],headers: Seq[(String,String)] = Seq.empty, auth: Option[(String, String)]): Task[(Int, String)] = Task.create[(Int,String)]{ (callback,_) =>
|
||||
println(s"Req: $method $url")
|
||||
val con = url.toURL.openConnection().asInstanceOf[HttpURLConnection]
|
||||
con.setRequestMethod(method)
|
||||
con.setRequestProperty("User-Agent",
|
||||
"Mozilla/5.0 (Windows NT 5.1; rv:19.0) Gecko/20100101 Firefox/19.0")
|
||||
headers.foreach{
|
||||
case (k,v) => con.setRequestProperty(k,v)
|
||||
}
|
||||
auth.foreach{ auth =>
|
||||
con.addRequestProperty("Authorization",Base64.getEncoder.encodeToString(s"${auth._1}:${auth._2}".getBytes("UTF-8")))
|
||||
}
|
||||
data.foreach(_ => con.setDoOutput(true))
|
||||
con.setDoInput(true)
|
||||
con.connect()
|
||||
data.foreach{ data =>
|
||||
val out = con.getOutputStream
|
||||
out.write(data.getBytes("UTF-8"))
|
||||
out.close()
|
||||
}
|
||||
callback.onSuccess((con.getResponseCode,Source.fromInputStream(con.getInputStream).getLines().mkString))
|
||||
new Cancelable {
|
||||
override def cancel(): Unit = con.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
override implicit def scheduler: Scheduler = Scheduler(executionContext)
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
/**
|
||||
* Created by marvin on 3/26/16.
|
||||
*/
|
||||
|
||||
|
||||
object JsonParser {
|
||||
|
||||
// def parse(json: String) = io.circe.parser.parse(json)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
/**
|
||||
* Created by marvin on 3/27/16.
|
||||
*/
|
||||
object UrlEncoder {
|
||||
|
||||
def encode(s: String,enc: String) = java.net.URLEncoder.encode(s,enc)
|
||||
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import java.net.URI
|
||||
|
||||
import cats.data.Xor
|
||||
import monix.async.Task
|
||||
import monix.execution.Scheduler
|
||||
|
||||
import scala.concurrent.{Promise, ExecutionContext, Future}
|
||||
import scala.util.{Try, Failure, Success}
|
||||
|
||||
/**
|
||||
* Created by marvin on 1/5/16.
|
||||
*/
|
||||
trait DataSource extends RedditLocation{
|
||||
|
||||
|
||||
private implicit def task2future[T](task: Task[T]): Future[T] = {
|
||||
val p = Promise[T]
|
||||
task.runAsync{ t: Try[T] =>
|
||||
t match {
|
||||
case Success(res) => p.success(res)
|
||||
case Failure(t) => p.failure(t)
|
||||
}
|
||||
}
|
||||
p.future
|
||||
}
|
||||
|
||||
@deprecated
|
||||
def get(url: URI): Future[(Int,Option[String])] = httpRequest(url).map(t => (t._1,Some(t._2)))
|
||||
|
||||
@deprecated
|
||||
def post(url: URI,body: String): Future[(Int,Option[String])] = httpRequest(url,"POST").map(t => (t._1,Some(t._2)))
|
||||
|
||||
def httpRequest(url: URL,method: String = "GET",data: Option[String] = None,headers: Seq[(String,String)] = Seq.empty,auth: Option[(String,String)] = None): Task[(Int,String)]
|
||||
|
||||
|
||||
def json(url: URL,auth: Option[(String,String)] = None,callbackParameter: String = "jsonp"): Task[(Int,String)] = httpRequest(url,auth=auth)
|
||||
|
||||
implicit def executionContext: ExecutionContext
|
||||
|
||||
implicit def scheduler: Scheduler
|
||||
|
||||
}
|
||||
|
||||
trait RedditLocation {
|
||||
|
||||
def secure = true
|
||||
|
||||
def domain: String = "reddit.com"
|
||||
|
||||
def authenticatedDomain: String = "oauth.reddit.com"
|
||||
|
||||
}
|
||||
|
||||
object DataSource extends DataSourceCompanion
|
||||
|
||||
142
api/shared/src/main/scala/net/shimun/redstream/api/Page.scala
Normal file
142
api/shared/src/main/scala/net/shimun/redstream/api/Page.scala
Normal file
@@ -0,0 +1,142 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import java.net.URI
|
||||
|
||||
import cats.Monad
|
||||
import cats.data.Xor
|
||||
import io.circe._
|
||||
import io.iteratee.{Enumeratee, Enumerator}
|
||||
import monix.async.Task
|
||||
import io.circe.parser.{parse => parseJSON}
|
||||
import net.shimun.redstream.api.models.Submission
|
||||
|
||||
/**
|
||||
* Created by marvin on 1/5/16.
|
||||
*/
|
||||
|
||||
case class Page(url: QueryURL, before: Option[String], after: Option[String], children: Seq[JsonObject],count: Int = 0) {
|
||||
|
||||
def next(implicit ds: DataSource): Enumerator[Task,Page] = after match {
|
||||
case Some(token) => Page.crawl(url,Some(Right(token)))
|
||||
case _ => Enumerator.empty[Task,Page]
|
||||
}
|
||||
|
||||
def previous(implicit ds: DataSource): Enumerator[Task,Page] = before match {
|
||||
case Some(token) => Page.crawl(url,Some(Left(token)))
|
||||
case _ => Enumerator.empty[Task,Page]
|
||||
}
|
||||
|
||||
override def toString: String = s"${getClass.getSimpleName}($url,$before,$after,..[${children.size}])"
|
||||
}
|
||||
|
||||
object Page {
|
||||
|
||||
private def defaultExtractor(obj: JsonObject): Seq[JsonObject] = (for{
|
||||
node <- obj("children").toSeq
|
||||
children <- node.asArray.toSeq
|
||||
child <- children
|
||||
childObj <- child.asObject
|
||||
} yield childObj)
|
||||
|
||||
def crawl(url: QueryURL, direction: Option[Either[String, String]] = None, limit: Int = 25,
|
||||
entrypoint: (Option[JsonObject]) => Option[JsonObject] = (obj) => obj ,extractor: (JsonObject) => Seq[JsonObject] = defaultExtractor)(implicit ds: DataSource): Enumerator[Task, Page] = {
|
||||
val baseURL = url.copy(parameters = Seq("limit" -> limit.toString) ++ (direction match {
|
||||
case Some(Left(token)) => Seq("before" -> token)
|
||||
case Some(Right(token)) => Seq("after" -> token)
|
||||
case _ => Seq.empty[(String, String)]
|
||||
}) ++ url.parameters.filterNot(p => Seq("count", "before", "after").contains(p._1)))
|
||||
import ds._
|
||||
def process(url: QueryURL,count: Int = 0): Task[Option[Page]] = {
|
||||
ds.json(url).flatMap {
|
||||
case (200, text) =>
|
||||
parseJSON(text) match {
|
||||
case Xor.Right(json: Json) =>
|
||||
Task.now {
|
||||
for {
|
||||
root <- entrypoint(json.asObject)
|
||||
kind <- root("kind").flatMap(_.asString)
|
||||
data <- root("data").flatMap(_.asObject) if kind == "Listing"
|
||||
_ <- data("modhash").flatMap(_.asString)
|
||||
after <- data("after").map(_.asString)
|
||||
before <- data("before").map(_.asString)
|
||||
} yield {
|
||||
val extracted = extractor(data)
|
||||
Page(url, before, after, extracted,count + extracted.size)
|
||||
}
|
||||
}
|
||||
case Xor.Left(failure) => Task.error(failure.underlying)
|
||||
}
|
||||
case (status, _) => Task.error(new RuntimeException(s"Failed to fetch page - $status"))
|
||||
}
|
||||
}
|
||||
|
||||
import taskModule._
|
||||
|
||||
val relativeDirection = direction match {
|
||||
case Some(Left(_)) => Left(())
|
||||
case _ => Right(())
|
||||
}
|
||||
|
||||
liftToEnumerator(process(baseURL))
|
||||
.flatMap(_.fold(Enumerator.empty[Task,Page])(Enumerator.enumOne(_)))
|
||||
.flatMap{ page =>
|
||||
iterateUntilM(page){ prev =>
|
||||
((relativeDirection,prev) match {
|
||||
case (Right(()),Page(_,_,Some(tkn),_,_)) => Some(baseURL.withParameters((baseURL.parameters :+ ("after" -> tkn)):_*))
|
||||
case (Left(()),Page(_,Some(tkn),_,_,_)) => Some(baseURL.withParameters((baseURL.parameters :+ ("before" -> tkn)):_*))
|
||||
case _ => None
|
||||
}).map(u=>u.withParameters((u.parameters :+ ("count" -> prev.count.toString)):_*)).map(process(_,prev.count)).getOrElse(Task.now(None))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
trait Paged {
|
||||
|
||||
def url(implicit rl: RedditLocation): QueryURL
|
||||
|
||||
def pages(implicit ds: DataSource): Enumerator[Task,Page] = Page.crawl(url(ds))(ds)
|
||||
|
||||
}
|
||||
|
||||
trait Owned[R] {
|
||||
|
||||
def parent: Either[User,R]
|
||||
|
||||
}
|
||||
|
||||
trait Rateable {
|
||||
|
||||
def id: String
|
||||
|
||||
def vote(up: Boolean)(implicit session: Session,ds: DataSource) = {
|
||||
import io.circe.syntax._
|
||||
import io.circe.generic.auto._
|
||||
case class Vote(dir: Int,id: String,rank: Int)
|
||||
session.authenticated(s"https://${ds.authenticatedDomain}/api/vote","POST",Some(Vote(if(up) 1 else -1,id,2).asJson.noSpaces))
|
||||
}
|
||||
|
||||
def up(implicit userSession: Session,ds: DataSource): Task[_ <: Rateable]
|
||||
|
||||
def down(implicit userSession: Session,ds: DataSource): Task[_ <: Rateable]
|
||||
|
||||
def ups: Int
|
||||
|
||||
//def downs: Int = ups - (ups * (upvoted / 100))
|
||||
|
||||
}
|
||||
|
||||
case class FormattedText(plain: String,html: String)
|
||||
|
||||
object FormattedText {
|
||||
val empty = FormattedText("","")
|
||||
}
|
||||
|
||||
case class Score(ups: Int,downs: Int) {
|
||||
def score = ups - downs
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
|
||||
/**
|
||||
* Created by marvin on 2/14/16.
|
||||
*/
|
||||
case class QueryURL(baseURL: URL,parameters: Seq[(String,String)] = Seq.empty) {
|
||||
|
||||
def toURL: URL = {
|
||||
import baseURL._
|
||||
new URL(getScheme,getUserInfo,getHost,getPort,getPath,parameters.toMap.toSeq.map{ kv =>
|
||||
val (key,value) = kv
|
||||
val encode = UrlEncoder.encode(_ : String,"UTF-8")
|
||||
s"${encode(key)}=${encode(value)}"
|
||||
}.mkString("","&",""),getFragment)
|
||||
}
|
||||
|
||||
def withParameters(parameters: (String,String)*) = copy(parameters = parameters)
|
||||
|
||||
override def toString: String = toURL.toString
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import monix.async.Task
|
||||
|
||||
/**
|
||||
* Created by marvin on 4/1/16.
|
||||
*/
|
||||
trait Session {
|
||||
|
||||
val api: Boolean
|
||||
|
||||
def authenticated(url: URL,method: String = "GET",data: Option[String] = None,headers: Seq[(String,String)] = Seq.empty)(implicit ds: DataSource): Task[(Int,String)]
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* {
|
||||
"access_token": Your access token,
|
||||
"token_type": "bearer",
|
||||
"expires_in": Unix Epoch Seconds,
|
||||
"scope": A scope string,
|
||||
"refresh_token": Your refresh token
|
||||
}
|
||||
* */
|
||||
|
||||
case class ApiSession(token: String,tokenType: String,expires: Long,scope: String,refreshToken: String) extends Session{
|
||||
|
||||
override val api: Boolean = true
|
||||
|
||||
override def authenticated(url: URL, method: String,data: Option[String],headers: Seq[(String,String)] = Seq.empty)(implicit ds: DataSource) =
|
||||
ds.httpRequest(url,method,data,headers :+ "Authorization" -> s"$tokenType $token")
|
||||
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
import monix.async.Task
|
||||
import net.shimun.redstream.api.models.{Comment, Submission}
|
||||
|
||||
//import scalaz.Id
|
||||
//import upickle.default._
|
||||
//
|
||||
//import scalaz.iteratee.Iteratee._
|
||||
//import scalaz.iteratee._
|
||||
|
||||
/**
|
||||
* Created by marvin on 1/5/16.
|
||||
*/
|
||||
|
||||
|
||||
|
||||
case class Subreddit(name: String,sorting: String = "hot") extends Paged {
|
||||
|
||||
|
||||
override def url(implicit rl: RedditLocation): QueryURL = QueryURL(new URL(s"http${if(rl.secure) "s" else ""}://${rl.domain}/r/$name/$sorting.json"))
|
||||
|
||||
def posts(implicit ds: DataSource) = {
|
||||
import ds._
|
||||
Submission(pages,this)
|
||||
}
|
||||
|
||||
def meta: Task[Option[Nothing]] = Task.now(None)
|
||||
|
||||
}
|
||||
|
||||
case class User(name: String) extends Paged {
|
||||
|
||||
override def url(implicit rl: RedditLocation): QueryURL = QueryURL(new URL(s"http${if(rl.secure) "s" else ""}://${rl.domain}/u/$name.json"))
|
||||
|
||||
def posts(implicit ds: DataSource) = {
|
||||
import ds._
|
||||
Submission(pages,this)
|
||||
}
|
||||
|
||||
def comments(implicit ds: DataSource) = {
|
||||
import ds._
|
||||
Comment(pages,this)
|
||||
}
|
||||
|
||||
def meta: Task[Option[Nothing]] = Task.now(None)
|
||||
|
||||
override def equals(obj: scala.Any): Boolean = obj match {
|
||||
case User(otherName) => otherName == name
|
||||
case other => other == name
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package net.shimun.redstream.api
|
||||
package models
|
||||
|
||||
import cats.{Applicative, Monad}
|
||||
import io.circe.JsonObject
|
||||
import io.iteratee.{Enumeratee, Enumerator}
|
||||
import monix.async.Task
|
||||
|
||||
/**
|
||||
* Created by marvin on 4/6/16.
|
||||
*/
|
||||
case class Comment(parent: Either[User,Submission],id: String,text: FormattedText,author: User, score: Int = 0,comments: Enumerator[Task,Comment] = Enumerator.empty) extends Rateable with Owned[Submission] {
|
||||
|
||||
override def ups: Int = score
|
||||
|
||||
override def up(implicit userSession: Session, ds: DataSource): Task[_ <: Rateable] = ???
|
||||
|
||||
override def down(implicit userSession: Session, ds: DataSource): Task[_ <: Rateable] = ???
|
||||
|
||||
def sub = parent match {
|
||||
case Right(sub) => Some(sub)
|
||||
case _ => None
|
||||
}
|
||||
|
||||
def profile = parent match {
|
||||
case Left(profile) => Some(profile)
|
||||
case _ => None
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
object Comment {
|
||||
|
||||
def apply(pages: Enumerator[Task, Page],parent: Submission): Enumerator[Task, Comment] =
|
||||
pages.flatMap(page => Enumerator.enumList[Task,JsonObject](page.children.toList)).mapE(parser(Right(parent)))
|
||||
|
||||
def apply(pages: Enumerator[Task, Page],parent: User): Enumerator[Task, Comment] =
|
||||
pages.flatMap(page => Enumerator.enumList[Task,JsonObject](page.children.toList)).mapE(parser(Left(parent)))
|
||||
|
||||
def parser(parent: Either[User,Submission]): Enumeratee[Task,JsonObject,Comment] = Enumeratee.flatMap[Task,JsonObject,Comment]{ obj =>
|
||||
trait Extractor[T] {
|
||||
def apply(o: JsonObject,name: String): Option[T]
|
||||
}
|
||||
implicit val str = new Extractor[String] {
|
||||
override def apply(o: JsonObject,name: String): Option[String] = o(name).flatMap(_.asString)
|
||||
}
|
||||
implicit val int = new Extractor[Int] {
|
||||
override def apply(o: JsonObject,name: String): Option[Int] = o(name).flatMap(_.asNumber.flatMap(_.toInt))
|
||||
}
|
||||
implicit def optStr = new Extractor[Option[String]] {
|
||||
override def apply(o: JsonObject,name: String): Option[Option[String]] = o(name).map(_.asString)
|
||||
}
|
||||
implicit val bool = new Extractor[Boolean] {
|
||||
override def apply(o: JsonObject,name: String): Option[Boolean] = o(name).flatMap(_.asBoolean)
|
||||
}
|
||||
def get[T](o: JsonObject,name: String)(implicit E: Extractor[T]) = E(o,name)
|
||||
|
||||
def parseReplies(replies: Option[JsonObject]): Enumerator[Task,Comment] =
|
||||
replies.fold(Enumerator.empty[Task,Comment]){ replies=>
|
||||
Enumerator.enumOne(replies).mapE(parser(parent))
|
||||
}
|
||||
|
||||
// val comments = for {
|
||||
// kind <- get[String](obj, "kind").toIndexedSeq
|
||||
// data <- obj("data").flatMap(_.asObject).toSeq if kind == "Listing"
|
||||
// before <- get[Option[String]](obj, "before")
|
||||
// after <- get[Option[String]](obj, "after")
|
||||
// _ <- get[Option[String]](obj, "modhash")
|
||||
// replies <- obj("replies").map(_.asObject)
|
||||
// children <- obj("children").flatMap(_.asArray)
|
||||
// childRoot <- children.flatMap(_.asObject.toSeq)
|
||||
// kind <- get[String](childRoot,"kind")
|
||||
// child <- childRoot("data").flatMap(_.asObject) if kind != "more"
|
||||
// id <- get[String](child, "id")
|
||||
// author <- get[String](child, "author")
|
||||
// body <- get[String](child, "body")
|
||||
// bodyHtml <- get[String](child, "body_html")
|
||||
// score <- get[Int](child, "score")
|
||||
// ups <- get[Int](child, "ups")
|
||||
// downs <- get[Int](child, "downs")
|
||||
// } yield Comment(parent, id, FormattedText(body, bodyHtml), User(author), score, parseReplies(replies))
|
||||
// Enumerator.enumIndexedSeq[Task,Comment](comments)
|
||||
???
|
||||
}
|
||||
|
||||
}
|
||||
@@ -5,4 +5,46 @@ package net.shimun.redstream.api.models
|
||||
*/
|
||||
object Parsing {
|
||||
|
||||
import java.net.URL
|
||||
import shapeless._
|
||||
|
||||
trait Convert[I <: HList, O <: HList] { def apply(i: I): O }
|
||||
|
||||
object Convert extends LowPriorityConvertInstances {
|
||||
implicit val convertHNil: Convert[HNil, HNil] = new Convert[HNil, HNil] {
|
||||
def apply(i: HNil): HNil = i
|
||||
}
|
||||
|
||||
implicit def convertHConsURL[T <: HList, TO <: HList](implicit
|
||||
c: Convert[T, TO]
|
||||
) = new Convert[String :: T, URL :: TO] {
|
||||
def apply(i: String :: T): URL :: TO = new URL(i.head) :: c(i.tail)
|
||||
}
|
||||
|
||||
implicit def convertHConsOptionalURL[T <: HList, TO <: HList](implicit
|
||||
c: Convert[T, TO]
|
||||
) = new Convert[Option[String] :: T, Option[URL] :: TO] {
|
||||
def apply(i: Option[String] :: T): Option[URL] :: TO = i.head.map(new URL(_)) :: c(i.tail)
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
sealed class LowPriorityConvertInstances {
|
||||
implicit def convertHCons[H, T <: HList, TO <: HList](implicit
|
||||
c: Convert[T, TO]
|
||||
): Convert[H :: T, H :: TO] = new Convert[H :: T, H :: TO] {
|
||||
def apply(i: H :: T): H :: TO = i.head :: c(i.tail)
|
||||
}
|
||||
}
|
||||
|
||||
class PartiallyAppliedConvert[O <: HList] {
|
||||
import Convert._
|
||||
def apply[I <: HList](i: I)(implicit c: Convert[I, O]): O = c(i)
|
||||
}
|
||||
|
||||
def convert[O <: HList]: PartiallyAppliedConvert[O] =
|
||||
new PartiallyAppliedConvert[O]
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
package net.shimun.redstream.api.models
|
||||
|
||||
import cats.Monad
|
||||
import io.circe.JsonObject
|
||||
import io.iteratee.{Enumeratee, Enumerator}
|
||||
import monix.async.Task
|
||||
import net.shimun.redstream.api._
|
||||
|
||||
/**
|
||||
* Created by marvin on 4/6/16.
|
||||
*/
|
||||
|
||||
case class Submission(parent: Either[User,Subreddit],title: String,id: String, author: User, domain: String, permalink: URL,self: Boolean, selfText: Option[FormattedText],thumbnail: Option[URL],url: Option[URL], nsfw: Boolean, commentCount: Int, score: Int) extends Rateable with Owned[Subreddit]{
|
||||
|
||||
override def ups: Int = score
|
||||
|
||||
def comments: Enumerator[Task, Comment] = ???
|
||||
|
||||
override def up(implicit userSession: Session, ds: DataSource): Task[_ <: Rateable] = vote(true).map{
|
||||
case (200,_) => copy(score=score+1)
|
||||
}
|
||||
|
||||
override def down(implicit userSession: Session, ds: DataSource): Task[_ <: Rateable] = vote(false).map{
|
||||
case (200,_) => copy(score=score-1)
|
||||
}
|
||||
|
||||
def sub = parent match {
|
||||
case Right(sub) => Some(sub)
|
||||
case _ => None
|
||||
}
|
||||
|
||||
def profile = parent match {
|
||||
case Left(profile) => Some(profile)
|
||||
case _ => None
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
object Submission {
|
||||
|
||||
case class RawSubmission(
|
||||
domain: String,
|
||||
banned_by: Option[String],
|
||||
subreddit: String,
|
||||
selftext_html: Option[String],
|
||||
selftext: Option[String],
|
||||
likes: Option[String],
|
||||
suggested_sort: Option[String],
|
||||
//user_reports: List[UserReport],
|
||||
secure_media: Option[String],
|
||||
link_flair_text: Option[String],
|
||||
id: String,
|
||||
from_kind: Option[String],
|
||||
gilded: Double,
|
||||
archived: Option[Boolean],
|
||||
clicked: Boolean,
|
||||
report_reasons: Option[String],
|
||||
author: String,
|
||||
media: Option[String],
|
||||
score: Double,
|
||||
approved_by: Option[String],
|
||||
over_18: Boolean,
|
||||
hidden: Boolean,
|
||||
num_comments: Int,
|
||||
thumbnail: String,
|
||||
subreddit_id: String,
|
||||
hide_score: Boolean,
|
||||
edited: Boolean,
|
||||
link_flair_css_class: Option[String],
|
||||
author_flair_css_class: Option[String],
|
||||
downs: Int,
|
||||
saved: Boolean,
|
||||
removal_reason: Option[String],
|
||||
stickied: Boolean,
|
||||
from: Option[String],
|
||||
is_self: Boolean,
|
||||
from_id: Option[String],
|
||||
permalink: String,
|
||||
locked: Boolean,
|
||||
name: String,
|
||||
created: Long,
|
||||
url: String,
|
||||
author_flair_text: Option[String],
|
||||
quarantine: Boolean,
|
||||
title: String,
|
||||
created_utc: Long,
|
||||
distinguished: Option[Boolean],
|
||||
//mod_reports: List[ModReport],
|
||||
visited: Boolean,
|
||||
num_reports: Option[Int],
|
||||
ups: Int
|
||||
)
|
||||
|
||||
def lift(raw: RawSubmission) = {
|
||||
import shapeless._
|
||||
import Parsing.convert
|
||||
val rawGen = Generic[RawSubmission]
|
||||
val gen = Generic[Submission]
|
||||
//gen.from(convert[gen.Repr](rawGen.to(raw)))
|
||||
}
|
||||
|
||||
def apply[F[_]](pages: Enumerator[F, Page],parent: Subreddit)(implicit F: Monad[F],rl: RedditLocation): Enumerator[F, Submission] =
|
||||
pages.flatMap(page => Enumerator.enumList[F,JsonObject](page.children.toList)).mapE(parser[F](Right(parent)))
|
||||
|
||||
def apply[F[_]](pages: Enumerator[F, Page],parent: User)(implicit F: Monad[F],rl: RedditLocation): Enumerator[F, Submission] =
|
||||
pages.flatMap(page => Enumerator.enumList[F,JsonObject](page.children.toList)).mapE(parser[F](Left(parent)))
|
||||
|
||||
//TODO: use circe + http://json2caseclass.cleverapps.io/ for full submission repr
|
||||
import io.circe._
|
||||
import io.circe.generic.auto._
|
||||
import io.circe.parser._
|
||||
def parser[F[_]](parent: Either[User,Subreddit])(implicit F: Monad[F],rl: RedditLocation) = Enumeratee.flatMap[F,JsonObject,Submission]{ obj =>
|
||||
Enumerator.enumList[F,Submission]((for {
|
||||
postData <- obj("data").flatMap(_.asObject)
|
||||
title <- postData("title").flatMap(_.asString)
|
||||
id <- postData("id").flatMap(_.asString)
|
||||
author <- postData("author").flatMap(_.asString)
|
||||
domain <- postData("domain").flatMap(_.asString)
|
||||
nsfw <- postData("over_18").flatMap(_.asBoolean)
|
||||
comments <- postData("num_comments").flatMap(_.asNumber)
|
||||
selftext <- postData("selftext").map(_.asString.filterNot(_ == ""))
|
||||
selftextHtml <- postData("selftext_html").map(_.asString.filterNot(_ == ""))
|
||||
url <- postData("url").map(_.asString)
|
||||
link <- postData("permalink").flatMap(_.asString).map(p=>s"http${if(rl.secure) "s" else ""}://${rl.domain}$p")
|
||||
isSelf <-postData("is_self").flatMap(_.asBoolean)
|
||||
thumbnail <- postData("thumbnail").map(_.asString.filterNot(_ == ""))
|
||||
score <- postData("score").flatMap(_.asNumber)
|
||||
} yield Submission(parent,title,id, User(author), domain, link,isSelf,selftext.flatMap(plain=>selftextHtml.map(FormattedText(plain,_))),thumbnail.map(url => url),url.map(url => url), nsfw, comments.truncateToInt, score.truncateToInt)).toList)
|
||||
}
|
||||
|
||||
object Common{
|
||||
def unapplySeq(submission: Submission) = {
|
||||
import submission._
|
||||
Some((parent,title,id,author,domain,permalink,self,selfText,thumbnail,url,nsfw,commentCount,score))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package net.shimun.redstream.api
|
||||
|
||||
/**
|
||||
* Created by marvin on 4/6/16.
|
||||
*/
|
||||
package object models {
|
||||
|
||||
sealed trait Report //TODO: populate
|
||||
|
||||
case class UserReport() extends Report
|
||||
|
||||
case class ModReport() extends Report
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package net.shimun.redstream
|
||||
|
||||
import java.net.{URI, URLDecoder}
|
||||
|
||||
import cats.Monad
|
||||
import io.iteratee.{Enumeratee, Iteratee, Module}
|
||||
import monix.async.Task
|
||||
import net.shimun.redstream.api.models.Submission
|
||||
|
||||
|
||||
/**
|
||||
* Created by marvin on 1/5/16.
|
||||
*/
|
||||
package object api {
|
||||
|
||||
type URL = java.net.URI
|
||||
|
||||
implicit def queryURL2URL(queryURL: QueryURL) = queryURL.toURL
|
||||
|
||||
implicit def URL2queryURL(url: URL) = {
|
||||
val decode = URLDecoder.decode(_: String,"UTF_8")
|
||||
QueryURL(url,url.getQuery.dropWhile(_ == '?').split("=").map(_.split("&").toSeq).collect{
|
||||
case Seq(key,value) => decode(key) -> decode(value)
|
||||
case Seq(key) => decode(key) -> ""
|
||||
})
|
||||
}
|
||||
|
||||
implicit def string2uri(str: String) = new URI(str)
|
||||
|
||||
implicit def TaskMonad = new Monad[Task] {
|
||||
|
||||
override def flatMap[A, B](fa: Task[A])(f: (A) => Task[B]): Task[B] = fa.flatMap(f)
|
||||
|
||||
override def pure[A](x: A): Task[A] = Task.now(x)
|
||||
}
|
||||
|
||||
private[api] val taskModule = new Module[Task] {}
|
||||
|
||||
def subreddit(name: String)(implicit ds: DataSource) = Subreddit(name)
|
||||
|
||||
def user(name: String)(implicit source: DataSource) = User(name)
|
||||
|
||||
def test = {
|
||||
implicit val j = DataSource.Default
|
||||
import j._
|
||||
subreddit("scalajs").posts.mapE(Enumeratee.take[Task,Submission](3)).run(Iteratee.foreach[Task,Submission]{ p =>
|
||||
println(s"Page: $p")
|
||||
}).runAsync
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user