From 2876dd7b24dc71e6a266906cd1ea5a2795e7fca3 Mon Sep 17 00:00:00 2001 From: Lucas Satabin Date: Thu, 3 Oct 2019 17:44:39 +0200 Subject: [PATCH] Bump dependency versions The behavior of scodec-stream changed quite a lot with the new fs2 2.0.0 version, which supports scala 2.13. This leads to breaking changes in the API, to make it work with new cats-effect and fs2 APIs. --- .scalafmt.conf | 2 +- .../src/swam/MandelbrotPerformances.scala | 20 ++++++++-- .../interpreter/StackPerformances.scala | 1 - build.sc | 15 ++++--- core/src/swam/ModuleLoader.scala | 30 +++++++++++--- core/src/swam/binary/ModuleStream.scala | 21 +--------- core/src/swam/binary/WasmCodec.scala | 40 +++++++++---------- core/src/swam/binary/package.scala | 6 +++ core/src/swam/decompilation/Decompiler.scala | 8 ++-- core/src/swam/opcodes.scala | 7 +++- core/src/swam/validation/Validator.scala | 3 +- runtime/src/swam/runtime/Engine.scala | 26 ++++++------ .../test/src/swam/runtime/ScriptEngine.scala | 10 ++--- runtime/test/src/swam/text/SpecTests.scala | 2 - 14 files changed, 107 insertions(+), 84 deletions(-) diff --git a/.scalafmt.conf b/.scalafmt.conf index 5efb0a60..1bc6ea5e 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = "2.0.0-RC7" +version = "2.0.0" maxColumn = 120 danglingParentheses = false align.openParenCallSite = true diff --git a/benchmarks/src/swam/MandelbrotPerformances.scala b/benchmarks/src/swam/MandelbrotPerformances.scala index 4b8ac635..d098ba12 100644 --- a/benchmarks/src/swam/MandelbrotPerformances.scala +++ b/benchmarks/src/swam/MandelbrotPerformances.scala @@ -21,10 +21,8 @@ import config._ import validation._ import formats.DefaultFormatters._ -import cats.implicits._ import cats.effect.IO -import pureconfig._ import pureconfig.generic.auto._ import pureconfig.module.squants._ import pureconfig.module.catseffect._ @@ -34,6 +32,10 @@ import org.openjdk.jmh.infra.Blackhole import java.nio.file.Paths import java.util.concurrent.TimeUnit +import cats.effect.Blocker +import scala.concurrent.ExecutionContext +import java.util.concurrent.Executors +import pureconfig.ConfigSource @BenchmarkMode(Array(Mode.AverageTime)) @OutputTimeUnit(TimeUnit.NANOSECONDS) @@ -58,18 +60,28 @@ class MandelbrotPerformances { private var mandelbrot: exports.EFunction4[Int, Double, Double, Double, Unit, IO] = _ + private val executor = Executors.newCachedThreadPool() + private val blocker = Blocker.liftExecutionContext(ExecutionContext.fromExecutor(executor)) + + implicit val cs = IO.contextShift(ExecutionContext.global) + @Setup def setup(): Unit = { mandelbrot = (for { v <- Validator[IO] - conf <- loadConfigF[IO, EngineConfiguration]("swam.runtime") + conf <- ConfigSource.default.at("swam.runtime").loadF[IO, EngineConfiguration] e = Engine[IO](conf.copy(useLowLevelAsm = useLowLevelAsm), v) - m <- e.compile(Paths.get("../../../../benchmarks/resources/mandelbrot.wasm")) + m <- e.compile(Paths.get("../../../../benchmarks/resources/mandelbrot.wasm"), blocker) i <- m.instantiate f <- i.exports.typed.procedure4[Int, Double, Double, Double]("mandelbrot") } yield f).unsafeRunSync() } + @TearDown + def teardown(): Unit = { + executor.shutdown() + } + @Benchmark def computeMandelbrot(bh: Blackhole): Unit = { bh.consume(mandelbrot(iterations, x, y, d).unsafeRunSync()) diff --git a/benchmarks/src/swam/runtime/internals/interpreter/StackPerformances.scala b/benchmarks/src/swam/runtime/internals/interpreter/StackPerformances.scala index fb171673..a10966b7 100644 --- a/benchmarks/src/swam/runtime/internals/interpreter/StackPerformances.scala +++ b/benchmarks/src/swam/runtime/internals/interpreter/StackPerformances.scala @@ -23,7 +23,6 @@ package high import org.openjdk.jmh.infra._ import org.openjdk.jmh.annotations._ -import cats._ import cats.implicits._ import scala.util.Random diff --git a/build.sc b/build.sc index b1affb6e..c3d36fe4 100644 --- a/build.sc +++ b/build.sc @@ -30,11 +30,13 @@ val swamUrl = "https://github.com/satabin/swam" val swamDeveloper = Developer("satabin", "Lucas Satabin", "https://github.com/satabin") -val pureconfigVersion = "0.11.1" +val fs2Version = "2.0.1" + +val pureconfigVersion = "0.12.1" trait SwamModule extends ScalaModule with ScalafmtModule with Headers { - def scalaVersion = "2.12.8" + def scalaVersion = "2.12.10" def scalacOptions = Seq("-feature", "-deprecation", "-unchecked", "-Ypartial-unification", "-Ypatmat-exhaust-depth", "off", "-Ywarn-unused:locals,imports") @@ -48,7 +50,7 @@ trait SwamModule extends ScalaModule with ScalafmtModule with Headers { trait ScoverageSwamModule extends SwamModule with ScoverageModule { - def scoverageVersion = "1.3.1" + def scoverageVersion = "1.4.0" } @@ -57,8 +59,9 @@ object core extends SwamModule with PublishModule { def ivyDeps = Agg( ivy"com.beachape::enumeratum:1.5.13", - ivy"co.fs2::fs2-core:1.1.0-M2", - ivy"org.scodec::scodec-stream:1.2.1", + ivy"co.fs2::fs2-core:$fs2Version", + ivy"co.fs2::fs2-io:$fs2Version", + ivy"org.scodec::scodec-stream:2.0.0", ivy"com.github.pureconfig::pureconfig-generic:$pureconfigVersion", ivy"com.github.pureconfig::pureconfig-squants:$pureconfigVersion", ivy"com.github.pureconfig::pureconfig-cats-effect:$pureconfigVersion", @@ -90,7 +93,7 @@ object core extends SwamModule with PublishModule { object text extends SwamModule with PublishModule { def moduleDeps = Seq(core) - def ivyDeps = Agg(ivy"com.lihaoyi::fastparse:2.1.3", ivy"co.fs2::fs2-io:1.1.0-M2") + def ivyDeps = Agg(ivy"com.lihaoyi::fastparse:2.1.3", ivy"co.fs2::fs2-io:$fs2Version") def publishVersion = swamVersion diff --git a/core/src/swam/ModuleLoader.scala b/core/src/swam/ModuleLoader.scala index bd23303a..ef008c0f 100644 --- a/core/src/swam/ModuleLoader.scala +++ b/core/src/swam/ModuleLoader.scala @@ -33,12 +33,32 @@ import scala.language.higherKinds */ class ModuleLoader[F[_]](implicit F: Effect[F]) { - def readPath(path: Path): Stream[F, Section] = - readBytes(BitVector.fromChannel(new java.io.FileInputStream(path.toFile).getChannel)) + def readPath(path: Path, blocker: Blocker, chunkSize: Int = 1024)(implicit cs: ContextShift[F]): Stream[F, Section] = + readBytes(io.file.readAll(path, blocker, chunkSize = chunkSize)) + + private val header = hex"0061736d01000000" /** Reads a binary module from the given bytes. */ - def readBytes(bytes: BitVector): Stream[F, Section] = - ModuleStream.decoder - .decode(bytes) + def readBytes(bytes: Stream[F, Byte]): Stream[F, Section] = { + def go(s: Stream[F, Byte]): Pull[F, Section, Unit] = + ModuleStream.decoder(s.chunks.map(_.toBitVector)).flatMap { + case Some(_) => Pull.raiseError(new BinaryException("unexpected end of input")) + case None => Pull.done + } + + bytes.pull + .unconsN(8, allowFewer = false) + .flatMap { + case Some((headerBytes, tl)) => + val bv = ByteVector(headerBytes.toArray) + if (bv == header) + Pull.pure(tl) + else + Pull.raiseError(new BinaryException(s"unexpected header ${bv.toHex(Bases.Alphabets.HexUppercase)}")) + case None => Pull.raiseError(new BinaryException("unexpected end of input")) + } + .flatMap(go(_)) + .stream + } } diff --git a/core/src/swam/binary/ModuleStream.scala b/core/src/swam/binary/ModuleStream.scala index 42dff511..885a10f0 100644 --- a/core/src/swam/binary/ModuleStream.scala +++ b/core/src/swam/binary/ModuleStream.scala @@ -19,13 +19,8 @@ package binary import syntax._ -import scodec.bits._ import scodec.stream._ -import scodec._ -import scodec.codecs._ -import scodec.codecs.literals._ - /** The module streams expose way to encode and decode WebAssembly modules in * binary format. * Section are streamed in the order they arrive, which may not be correct when @@ -34,22 +29,10 @@ import scodec.codecs.literals._ */ object ModuleStream { - private val header: Codec[Unit] = - ("magic" | hex"0061736d") ~> - ("version" | hex"01000000") - - val sections: StreamCodec[Section] = - StreamCodec - .instance(encode.once(WasmCodec.section), decode.once(WasmCodec.section)) - .many - val decoder: StreamDecoder[Section] = - for { - () <- decode.once(header) - section <- sections - } yield section + StreamDecoder.many(WasmCodec.section) def encoder: StreamEncoder[Section] = - encode.emit(hex"0061736d01000000".bits) ++ sections + StreamEncoder.many(WasmCodec.section) } diff --git a/core/src/swam/binary/WasmCodec.scala b/core/src/swam/binary/WasmCodec.scala index 39193909..0f020a38 100644 --- a/core/src/swam/binary/WasmCodec.scala +++ b/core/src/swam/binary/WasmCodec.scala @@ -26,17 +26,17 @@ import scodec.codecs._ object WasmCodec extends InstCodec { - protected val externalKind: Codec[ExternalKind] = + val externalKind: Codec[ExternalKind] = mappedEnum(byte, Map[ExternalKind, Byte](ExternalKind.Function -> 0, ExternalKind.Table -> 1, ExternalKind.Memory -> 2, ExternalKind.Global -> 3)) - protected val types: Codec[Vector[FuncType]] = + val types: Codec[Vector[FuncType]] = variableSizeBytes(varuint32, vectorWithN(varuint32, funcType)) - protected val importEntry: Codec[String ~ String ~ Import] = + val importEntry: Codec[String ~ String ~ Import] = (("module" | variableSizeBytes(varuint32, utf8)) ~ ("field" | variableSizeBytes(varuint32, utf8))).flatZip { case (module, field) => @@ -50,65 +50,65 @@ object WasmCodec extends InstCodec { .|(ExternalKind.Global) { case Import.Global(_, _, tpe) => tpe }(Import.Global(module, field, _))(globalType) } - protected val imports: Codec[Vector[Import]] = + val imports: Codec[Vector[Import]] = variableSizeBytes(varuint32, vectorWithN(varuint32, importEntry.xmap({ case (_, e) => e }, { case e @ Import(mod, fld) => ((mod, fld), e) }))) - protected val functions: Codec[Vector[Int]] = + val functions: Codec[Vector[Int]] = variableSizeBytes(varuint32, vectorWithN(varuint32, varuint32)) - protected val tables: Codec[Vector[TableType]] = + val tables: Codec[Vector[TableType]] = variableSizeBytes(varuint32, vectorWithN(varuint32, tableType)) - protected val mems: Codec[Vector[MemType]] = + val mems: Codec[Vector[MemType]] = variableSizeBytes(varuint32, vectorWithN(varuint32, memoryType)) - protected val globalVariable: Codec[Global] = + val globalVariable: Codec[Global] = (globalType :: expr).as[Global] - protected val globals: Codec[Vector[Global]] = + val globals: Codec[Vector[Global]] = variableSizeBytes(varuint32, vectorWithN(varuint32, globalVariable)) - protected val elemSegment: Codec[Elem] = + val elemSegment: Codec[Elem] = (("index" | varuint32) :: ("offset" | expr) :: ("elems" | vectorOfN(varuint32, varuint32))).as[Elem] - protected val elem: Codec[Vector[Elem]] = + val elem: Codec[Vector[Elem]] = variableSizeBytes(varuint32, vectorWithN(varuint32, elemSegment)) - protected val dataSegment: Codec[Data] = + val dataSegment: Codec[Data] = (("index" | varuint32) :: ("offset" | expr) :: ("data" | variableSizeBytes(varuint32, scodec.codecs.bits))).as[Data] - protected val data: Codec[Vector[Data]] = + val data: Codec[Vector[Data]] = variableSizeBytes(varuint32, vectorWithN(varuint32, dataSegment)) - protected val start: Codec[FuncIdx] = + val start: Codec[FuncIdx] = variableSizeBytes(varuint32, varuint32) - protected val localEntry: Codec[LocalEntry] = + val localEntry: Codec[LocalEntry] = (varuint32 :: valType).as[LocalEntry] - protected val funcBody: Codec[FuncBody] = + val funcBody: Codec[FuncBody] = variableSizeBytes(varuint32, (("locals" | vectorOfN(varuint32, localEntry)) :: ("code" | expr)).as[FuncBody]) - protected val code: Codec[Vector[FuncBody]] = + val code: Codec[Vector[FuncBody]] = variableSizeBytes(varuint32, vectorWithN(varuint32, funcBody)) - protected val exportEntry: Codec[Export] = + val exportEntry: Codec[Export] = (("field" | variableSizeBytes(varuint32, utf8)) :: ("kind" | externalKind) :: ("index" | varuint32)).as[Export] - protected val exports: Codec[Vector[Export]] = + val exports: Codec[Vector[Export]] = variableSizeBytes(varuint32, vectorWithN(varuint32, exportEntry)) - protected val custom: Codec[(String, BitVector)] = + val custom: Codec[(String, BitVector)] = variableSizeBytes(varuint32, (("name" | variableSizeBytes(varuint32, utf8)) ~ ("payload" | scodec.codecs.bits))) diff --git a/core/src/swam/binary/package.scala b/core/src/swam/binary/package.scala index 6d5c1781..d36f5257 100644 --- a/core/src/swam/binary/package.scala +++ b/core/src/swam/binary/package.scala @@ -16,6 +16,8 @@ package swam +import fs2._ + import scodec.bits._ import scodec._ import scodec.codecs._ @@ -24,8 +26,12 @@ import scala.annotation.tailrec import scala.collection.immutable.VectorBuilder +import scala.language.higherKinds + package object binary { + type VarResut[F[_]] = (Long, Option[(ByteVector, Int, Stream[F, Byte])]) + val noop: Codec[Unit] = new Codec[Unit] { def decode(bits: BitVector): Attempt[DecodeResult[Unit]] = Attempt.successful(DecodeResult((), bits)) diff --git a/core/src/swam/decompilation/Decompiler.scala b/core/src/swam/decompilation/Decompiler.scala index 9d0c93b7..49ec9c9d 100644 --- a/core/src/swam/decompilation/Decompiler.scala +++ b/core/src/swam/decompilation/Decompiler.scala @@ -22,8 +22,6 @@ import util.pretty.Doc import cats.effect._ -import scodec.bits._ - import fs2._ import java.nio.file.Path @@ -40,15 +38,15 @@ abstract class Decompiler[F[_]](implicit F: Effect[F]) extends ModuleLoader[F] { * * The module is not validated so invalid modules can also be decompiled. */ - def decompile(path: Path): F[Doc] = - decompile(readPath(path)) + def decompilePath(path: Path, blocker: Blocker, chunkSize: Int = 1024)(implicit cs: ContextShift[F]): F[Doc] = + decompile(readPath(path, blocker, chunkSize)) /** Returns a pretty-printed [[swam.util.pretty.Doc Doc]] resulting from decompiling * the module at the given bytes. * * The module is not validated so invalid modules can also be decompiled. */ - def decompile(bytes: BitVector): F[Doc] = + def decompileBytes(bytes: Stream[F, Byte]): F[Doc] = decompile(readBytes(bytes)) /** Returns a pretty-printed [[swam.util.pretty.Doc Doc]] resulting from decompiling diff --git a/core/src/swam/opcodes.scala b/core/src/swam/opcodes.scala index a6b207ab..2cca8583 100644 --- a/core/src/swam/opcodes.scala +++ b/core/src/swam/opcodes.scala @@ -196,8 +196,11 @@ object OpCode { final val F64ReinterpretI64 = 0xbf def withValueOpt(i: Int): Option[OpCode] = - if (all.contains(i)) - Some(i) + unapply(i.toByte) + + def unapply(b: Byte): Option[OpCode] = + if (all.contains(b)) + Some(b) else None diff --git a/core/src/swam/validation/Validator.scala b/core/src/swam/validation/Validator.scala index ebbdbed7..be690b5a 100644 --- a/core/src/swam/validation/Validator.scala +++ b/core/src/swam/validation/Validator.scala @@ -24,6 +24,7 @@ import cats._ import cats.effect._ import cats.implicits._ +import pureconfig._ import pureconfig.generic.auto._ import pureconfig.module.squants._ import pureconfig.module.catseffect._ @@ -52,6 +53,6 @@ object Validator { def apply[F[_]: Sync]: F[Validator[F]] = for { - conf <- loadConfigF[F, ValidationConfiguration]("swam.validation") + conf <- ConfigSource.default.at("swam.validation").loadF[F, ValidationConfiguration] } yield apply[F](conf) } diff --git a/runtime/src/swam/runtime/Engine.scala b/runtime/src/swam/runtime/Engine.scala index 9826e716..3c70ea44 100644 --- a/runtime/src/swam/runtime/Engine.scala +++ b/runtime/src/swam/runtime/Engine.scala @@ -32,8 +32,7 @@ import cats.effect._ import fs2._ -import scodec.bits._ - +import pureconfig._ import pureconfig.generic.auto._ import pureconfig.module.squants._ import pureconfig.module.catseffect._ @@ -70,14 +69,14 @@ class Engine[F[_]: Effect] private (val conf: EngineConfiguration, private[runti * * If validation fails, returns an error with the validation message wrapped in it. */ - def validate(path: Path): F[Unit] = - validate(readPath(path)) + def validate(path: Path, blocker: Blocker, chunkSize: Int = 1024)(implicit cs: ContextShift[F]): F[Unit] = + validate(readPath(path, blocker, chunkSize)) /** Reads the given binary encoded module and validates it. * * If validation fails, returns an error with the validation message wrapped in it. */ - def validate(bytes: BitVector): F[Unit] = + def validateBytes(bytes: Stream[F, Byte]): F[Unit] = validate(readBytes(bytes)) /** Reads the given stream of binary module sections and validates it. @@ -96,8 +95,8 @@ class Engine[F[_]: Effect] private (val conf: EngineConfiguration, private[runti * If validation or compilation fails, returns an error with the * message wrapped in it. */ - def compile(path: Path): F[Module[F]] = - compile(readPath(path)) + def compile(path: Path, blocker: Blocker, chunkSize: Int = 1024)(implicit cs: ContextShift[F]): F[Module[F]] = + compile(readPath(path, blocker, chunkSize)) /** Reads the given binary encoded module, validates, and compiles it. * The returned compiled [[Module]] can then be instantiated to be run. @@ -105,7 +104,7 @@ class Engine[F[_]: Effect] private (val conf: EngineConfiguration, private[runti * If validation or compilation fails, returns an error with the * message wrapped in it. */ - def compile(bytes: BitVector): F[Module[F]] = + def compileBytes(bytes: Stream[F, Byte]): F[Module[F]] = compile(readBytes(bytes)) /** Reads the given stream of binary module sections, validates, and compiles it. @@ -128,8 +127,9 @@ class Engine[F[_]: Effect] private (val conf: EngineConfiguration, private[runti * If validation, compilation, or instantiation fails, returns an error with the * message wrapped in it. */ - def instantiate(path: Path, imports: Imports[F]): F[Instance[F]] = - instantiate(readPath(path), imports) + def instantiate(path: Path, imports: Imports[F], blocker: Blocker, chunkSize: Int = 1024)( + implicit cs: ContextShift[F]): F[Instance[F]] = + instantiate(readPath(path, blocker, chunkSize), imports) /** Reads the given binary encoded module, validates, compiles, and instantiates it. * The returned [[Instance]] can then be used to access exported elements. @@ -137,8 +137,8 @@ class Engine[F[_]: Effect] private (val conf: EngineConfiguration, private[runti * If validation, compilation, or instantiation fails, returns an error with the * message wrapped in it. */ - def instantiate(bytes: BitVector, imports: Imports[F]): F[Instance[F]] = - compile(bytes).flatMap(instantiate(_, imports)) + def instantiateBytes(bytes: Stream[F, Byte], imports: Imports[F]): F[Instance[F]] = + compileBytes(bytes).flatMap(instantiate(_, imports)) /** Reads the given stream of binary module sections, validates, compiles, and instantiates it. * The returned [[Instance]] can then be used to access exported elements. @@ -164,7 +164,7 @@ object Engine { def apply[F[_]: Effect](): F[Engine[F]] = for { validator <- Validator[F] - conf <- loadConfigF[F, EngineConfiguration]("swam.runtime") + conf <- ConfigSource.default.at("swam.runtime").loadF[F, EngineConfiguration] } yield new Engine[F](conf, validator) def apply[F[_]: Effect](conf: EngineConfiguration, validator: Validator[F]): Engine[F] = diff --git a/runtime/test/src/swam/runtime/ScriptEngine.scala b/runtime/test/src/swam/runtime/ScriptEngine.scala index 1cc07ac1..377c2997 100644 --- a/runtime/test/src/swam/runtime/ScriptEngine.scala +++ b/runtime/test/src/swam/runtime/ScriptEngine.scala @@ -25,15 +25,15 @@ import imports._ import unresolved._ import validation._ - import cats._ import cats.implicits._ import cats.effect._ -import scodec.bits._ +import fs2._ import java.lang.{Float => JFloat, Double => JDouble} +import pureconfig._ import pureconfig.generic.auto._ import pureconfig.module.squants._ import pureconfig.module.catseffect._ @@ -76,7 +76,7 @@ class ScriptEngine(useLowLevel: Boolean) { val engine = for { validator <- Validator[IO] - conf <- loadConfigF[IO, EngineConfiguration]("swam.runtime") + conf <- ConfigSource.default.at("swam.runtime").loadF[IO, EngineConfiguration] } yield Engine[IO](conf.copy(useLowLevelAsm = useLowLevel), validator) val tcompiler = Compiler[IO] @@ -100,7 +100,7 @@ class ScriptEngine(useLowLevel: Boolean) { for { engine <- engine tcompiler <- tcompiler - compiled <- engine.compile(BitVector(bs)) + compiled <- engine.compileBytes(Stream.emits(bs.toArray)) instance <- compiled.importing(ctx.imports).instantiate } yield id match { case Some(name) => @@ -254,7 +254,7 @@ class ScriptEngine(useLowLevel: Boolean) { for { engine <- engine tcompiler <- tcompiler - res <- engine.compile(BitVector(bs)) + res <- engine.compileBytes(Stream.emits(bs.toArray)) } yield res case QuotedModule(_, src) => for { diff --git a/runtime/test/src/swam/text/SpecTests.scala b/runtime/test/src/swam/text/SpecTests.scala index 88ab62dc..e2b443ce 100644 --- a/runtime/test/src/swam/text/SpecTests.scala +++ b/runtime/test/src/swam/text/SpecTests.scala @@ -17,10 +17,8 @@ package swam package text -import util._ import parser._ import runtime._ -import validation._ import swam.test.util._