Expose overview strategy into Layers configuration (#252)

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - WMS GetMap support of extended parameters [#236](/~https://github.com/geotrellis/geotrellis-server/issues/236)
 - WCS GetCoverage support of extended parameters [#238](/~https://github.com/geotrellis/geotrellis-server/issues/238)
 - ColorRampStyle.clampWithColor option to render colors outside the requested render range as colors in the ramp instead of as transparent pixels [#220](/~https://github.com/geotrellis/geotrellis-server/issues/220)
+- Expose overview strategy into the layers configuration [#252](/~https://github.com/geotrellis/geotrellis-server/pull/252)
 
 ### Changed
 
@@ -24,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Addressed GeoTrellisRasterSourceLegacy issues and minimized number of RasterSource instances constructed for GeoTrellis Layers [#219](/~https://github.com/geotrellis/geotrellis-server/issues/219)
+- Some source resolutions are sometimes skipped leading to reading too much tiles [#215](/~https://github.com/geotrellis/geotrellis-server/issues/215)
 
 ## [4.1.0] - 2020-03-03
 

core/src/main/scala/geotrellis/server/vlm/RasterSourceUtils.scala

@@ -19,6 +19,7 @@ package geotrellis.server.vlm
 import geotrellis.proj4.{CRS, WebMercator}
 import geotrellis.raster._
 import geotrellis.raster.resample._
+import geotrellis.raster.io.geotiff._
 import geotrellis.layer._
 
 import cats.effect.IO
@@ -27,6 +28,8 @@ import _root_.io.circe.{Decoder, Encoder}
 
 import java.net.URI
 
+import scala.util.Try
+
 trait RasterSourceUtils {
   implicit val cellTypeEncoder: Encoder[CellType] = Encoder.encodeString.contramap[CellType](CellType.toName)
   implicit val cellTypeDecoder: Decoder[CellType] = Decoder[String].emap { name => Right(CellType.fromName(name)) }
@@ -64,6 +67,30 @@ trait RasterSourceUtils {
       case "sum"               => Sum
     }
 
+  implicit val overviewStrategyEncoder: Encoder[OverviewStrategy] =
+    Encoder.encodeString.contramap[OverviewStrategy] {
+      case AutoHigherResolution => "auto-higher-resolution"
+      case Base                 => "base"
+      case Auto(n)              => s"auto-$n"
+      case Level(n)             => s"level-$n"
+    }
+
+  implicit val overviewStrategyDecoder: Decoder[OverviewStrategy] = {
+    def parse(strategy: String, input: String): OverviewStrategy =
+      Auto(Try { input.split(s"$strategy-").last.toInt }.toOption.getOrElse(0))
+
+    def parseAuto(str: String): OverviewStrategy  = parse("auto", str)
+    def parseLevel(str: String): OverviewStrategy = parse("level", str)
+
+    Decoder.decodeString.map {
+      case "auto-higher-resolution"       => AutoHigherResolution
+      case "base"                         => Base
+      case str if str.startsWith("auto")  => parseAuto(str)
+      case str if str.startsWith("level") => parseLevel(str)
+      case _                              => OverviewStrategy.DEFAULT
+    }
+  }
+
   def getRasterSource(uri: String): RasterSource
 
   // the target CRS
@@ -74,13 +101,22 @@ trait RasterSourceUtils {
     for (zoom <- 0 to 64) yield scheme.levelForZoom(zoom).layout
   }.toArray
 
-  def fetchTile(uri: String, zoom: Int, x: Int, y: Int, crs: CRS = WebMercator, method: ResampleMethod = NearestNeighbor, target: ResampleTarget = DefaultTarget): IO[Raster[MultibandTile]] =
+  def fetchTile(
+    uri: String,
+    zoom: Int,
+    x: Int,
+    y: Int,
+    crs: CRS = WebMercator,
+    method: ResampleMethod = ResampleMethod.DEFAULT,
+    target: ResampleTarget = DefaultTarget,
+    overviewStrategy: OverviewStrategy = OverviewStrategy.DEFAULT
+  ): IO[Raster[MultibandTile]] =
     IO {
       val key = SpatialKey(x, y)
       val ld = tmsLevels(zoom)
       val rs = getRasterSource(uri)
         .reproject(crs, target)
-        .tileToLayout(ld, method)
+        .tileToLayout(ld, identity, method, overviewStrategy)
 
       rs.read(key).map(Raster(_, ld.mapTransform(key)))
     } flatMap {

core/src/main/scala/geotrellis/server/vlm/geotiff/GeoTiffNode.scala

@@ -20,9 +20,9 @@ import geotrellis.server._
 import geotrellis.server.vlm._
 import geotrellis.proj4.WebMercator
 import geotrellis.raster._
-import geotrellis.raster.resample.NearestNeighbor
+import geotrellis.raster.resample.ResampleMethod
 import geotrellis.raster.geotiff.GeoTiffRasterSource
-import geotrellis.raster.io.geotiff.AutoHigherResolution
+import geotrellis.raster.io.geotiff.OverviewStrategy
 import geotrellis.vector.Extent
 
 import _root_.io.circe._
@@ -32,7 +32,13 @@ import cats.data.{NonEmptyList => NEL}
 
 import java.net.URI
 
-case class GeoTiffNode(uri: URI, band: Int, celltype: Option[CellType], resampleMethod: ResampleMethod = NearestNeighbor)
+case class GeoTiffNode(
+  uri: URI,
+  band: Int,
+  celltype: Option[CellType],
+  resampleMethod: ResampleMethod = ResampleMethod.DEFAULT,
+  overviewStrategy: OverviewStrategy = OverviewStrategy.DEFAULT
+)
 
 object GeoTiffNode extends RasterSourceUtils {
   def getRasterSource(uri: String): GeoTiffRasterSource = GeoTiffRasterSource(uri)
@@ -48,7 +54,7 @@ object GeoTiffNode extends RasterSourceUtils {
   implicit val cogNodeTmsReification: TmsReification[GeoTiffNode] = new TmsReification[GeoTiffNode] {
     def tmsReification(self: GeoTiffNode, buffer: Int)(implicit contextShift: ContextShift[IO]): (Int, Int, Int) => IO[ProjectedRaster[MultibandTile]] = (z: Int, x: Int, y: Int) => {
       def fetch(xCoord: Int, yCoord: Int) =
-        fetchTile(self.uri.toString, z, xCoord, yCoord, WebMercator)
+        fetchTile(self.uri.toString, z, xCoord, yCoord, WebMercator, method = self.resampleMethod, overviewStrategy = self.overviewStrategy)
           .map(_.tile)
           .map(_.band(self.band))
 
@@ -62,7 +68,7 @@ object GeoTiffNode extends RasterSourceUtils {
   implicit val CogNodeExtentReification: ExtentReification[GeoTiffNode] = new ExtentReification[GeoTiffNode] {
     def extentReification(self: GeoTiffNode)(implicit contextShift: ContextShift[IO]): (Extent, CellSize) => IO[ProjectedRaster[MultibandTile]] = (extent: Extent, cs: CellSize) => {
       getRasterSource(self.uri.toString)
-        .resample(TargetRegion(new GridExtent[Long](extent, cs)), self.resampleMethod, AutoHigherResolution)
+        .resample(TargetRegion(new GridExtent[Long](extent, cs)), self.resampleMethod, self.overviewStrategy)
         .read(extent, self.band :: Nil)
         .map { ProjectedRaster(_, WebMercator) }
         .toIO { new Exception(s"No tile avail for RasterExtent: ${RasterExtent(extent, cs)}") }

core/src/test/scala/geotrellis/server/ResourceTile.scala

@@ -19,18 +19,19 @@ package geotrellis.server
 import geotrellis.server.vlm._
 import geotrellis.raster._
 import geotrellis.raster.geotiff._
-import geotrellis.raster.io.geotiff.AutoHigherResolution
-import geotrellis.raster.resample.NearestNeighbor
+import geotrellis.raster.io.geotiff.OverviewStrategy
+import geotrellis.raster.resample._
 import geotrellis.vector.Extent
 
 import cats.effect._
 import cats.data.{NonEmptyList => NEL}
 
-case class ResourceTile(name: String, resampleMethod: ResampleMethod = NearestNeighbor) {
-  def uri = {
-    val f = getClass.getResource(s"/$name").getFile
-    s"file://$f"
-  }
+case class ResourceTile(
+  name: String,
+  resampleMethod: ResampleMethod = ResampleMethod.DEFAULT,
+  overviewStrategy: OverviewStrategy = OverviewStrategy.DEFAULT
+) {
+  def uri: String = s"file://${getClass.getResource(s"/$name").getFile}"
 }
 
 object ResourceTile extends RasterSourceUtils {
@@ -40,7 +41,7 @@ object ResourceTile extends RasterSourceUtils {
     def extentReification(self: ResourceTile)(implicit contextShift: ContextShift[IO]): (Extent, CellSize) => IO[ProjectedRaster[MultibandTile]] =
       (extent: Extent, cs: CellSize) => {
         val rs = getRasterSource(self.uri.toString)
-        rs.resample(TargetRegion(new GridExtent[Long](extent, cs)), self.resampleMethod, AutoHigherResolution)
+        rs.resample(TargetRegion(new GridExtent[Long](extent, cs)), self.resampleMethod, self.overviewStrategy)
           .read(extent)
           .map { raster => ProjectedRaster(raster, rs.crs) }
           .toIO { new Exception(s"No tile avail for RasterExtent: ${RasterExtent(extent, cs)}") }

ogc-example/src/main/resources/application.conf

@@ -349,6 +349,7 @@ layers = {
       name = "us-ned-hillshade"
       title = "US NED Hillshade"
       resample-method = "bilinear"
+      overview-strategy = "auto-0"
       algebra = {
         "args" : [
           {

ogc-example/src/main/scala/geotrellis/server/ogc/Main.scala

@@ -107,7 +107,7 @@ object Main extends CommandApp(
             simpleSources = conf
               .layers
               .values
-              .collect { case rsc@RasterSourceConf(_, _, _, _, _, _) => rsc.toLayer }
+              .collect { case rsc@RasterSourceConf(_, _, _, _, _, _, _) => rsc.toLayer }
               .toList
             _ <- Stream.eval(IO(logOptState(
               conf.wms,

ogc-example/src/main/scala/geotrellis/server/ogc/conf/OgcServiceConf.scala

@@ -30,9 +30,9 @@ sealed trait OgcServiceConf {
   def layerDefinitions: List[OgcSourceConf]
   def layerSources(rasterOgcSources: List[RasterOgcSource]): OgcSourceRepository = {
     val rasterLayers: List[RasterOgcSource] =
-      layerDefinitions.collect { case rsc @ RasterSourceConf(_, _, _, _, _, _) => rsc.toLayer }
+      layerDefinitions.collect { case rsc @ RasterSourceConf(_, _, _, _, _, _, _) => rsc.toLayer }
     val mapAlgebraLayers: List[MapAlgebraSource] =
-      layerDefinitions.collect { case masc @ MapAlgebraSourceConf(_, _, _, _, _, _) => masc.model(rasterOgcSources) }
+      layerDefinitions.collect { case masc @ MapAlgebraSourceConf(_, _, _, _, _, _, _) => masc.model(rasterOgcSources) }
     ogc.OgcSourceRepository(rasterLayers ++ mapAlgebraLayers)
   }
 }

ogc-example/src/main/scala/geotrellis/server/ogc/conf/OgcSourceConf.scala

@@ -16,11 +16,12 @@
 
 package geotrellis.server.ogc.conf
 
-import com.azavea.maml.ast._
-import geotrellis.raster.{RasterSource, ResampleMethod}
-import geotrellis.raster.resample.NearestNeighbor
+import geotrellis.raster.RasterSource
+import geotrellis.raster.io.geotiff.OverviewStrategy
+import geotrellis.raster.resample._
 import geotrellis.server.ogc._
 import geotrellis.store.GeoTrellisPath
+import com.azavea.maml.ast._
 
 // This sumtype corresponds to the in-config representation of a source
 sealed trait OgcSourceConf {
@@ -35,15 +36,16 @@ case class RasterSourceConf(
   source: String,
   defaultStyle: Option[String],
   styles: List[StyleConf],
-  resampleMethod: ResampleMethod = NearestNeighbor
+  resampleMethod: ResampleMethod = ResampleMethod.DEFAULT,
+  overviewStrategy: OverviewStrategy = OverviewStrategy.DEFAULT
 ) extends OgcSourceConf {
   def toLayer: RasterOgcSource = {
     GeoTrellisPath.parseOption(source) match {
       case Some(_) => GeoTrellisOgcSource(
-        name, title, source, defaultStyle, styles.map(_.toStyle), resampleMethod
+        name, title, source, defaultStyle, styles.map(_.toStyle), resampleMethod, overviewStrategy
       )
       case None => SimpleSource(
-        name, title, RasterSource(source), defaultStyle, styles.map(_.toStyle), resampleMethod
+        name, title, RasterSource(source), defaultStyle, styles.map(_.toStyle), resampleMethod, overviewStrategy
       )
     }
   }
@@ -55,7 +57,8 @@ case class MapAlgebraSourceConf(
   algebra: Expression,
   defaultStyle: Option[String],
   styles: List[StyleConf],
-  resampleMethod: ResampleMethod = NearestNeighbor
+  resampleMethod: ResampleMethod = ResampleMethod.DEFAULT,
+  overviewStrategy: OverviewStrategy = OverviewStrategy.DEFAULT
 ) extends OgcSourceConf {
   private def listParams(expr: Expression): List[String] = {
     def eval(subExpr: Expression): List[String] = subExpr match {
@@ -81,6 +84,6 @@ case class MapAlgebraSourceConf(
       }
       name -> layerSrc.source
     }
-    MapAlgebraSource(name, title, sourceList.toMap, algebra, defaultStyle, styles.map(_.toStyle), resampleMethod)
+    MapAlgebraSource(name, title, sourceList.toMap, algebra, defaultStyle, styles.map(_.toStyle), resampleMethod, overviewStrategy)
   }
 }

ogc-example/src/main/scala/geotrellis/server/ogc/conf/package.scala

@@ -22,10 +22,10 @@ import geotrellis.proj4.CRS
 import geotrellis.vector.Extent
 import geotrellis.raster.{ResampleMethod, TileLayout, resample}
 import geotrellis.raster.render.{ColorMap, ColorRamp}
-
 import com.azavea.maml.ast._
 import com.azavea.maml.ast.codec.tree._
 import com.typesafe.config._
+import geotrellis.raster.io.geotiff.{Auto, AutoHigherResolution, Base, OverviewStrategy}
 import io.circe._
 import io.circe.parser._
 import pureconfig._
@@ -167,6 +167,22 @@ package object conf {
       case "sum"               => resample.Sum
     }
 
+  implicit val overviewStrategyReader: ConfigReader[OverviewStrategy] = {
+    def parse(strategy: String, input: String): OverviewStrategy =
+      Auto(Try { input.split(s"$strategy-").last.toInt }.toOption.getOrElse(0))
+
+    def parseAuto(str: String): OverviewStrategy  = parse("auto", str)
+    def parseLevel(str: String): OverviewStrategy = parse("level", str)
+
+    ConfigReader[String].map {
+      case "auto-higher-resolution"       => AutoHigherResolution
+      case "base"                         => Base
+      case str if str.startsWith("auto")  => parseAuto(str)
+      case str if str.startsWith("level") => parseLevel(str)
+      case _                              => OverviewStrategy.DEFAULT
+    }
+  }
+
   /** An alternative AST reading strategy that uses a separate json file */
   //private lazy val s3client = AmazonS3ClientBuilder.defaultClient()
 

ogc-example/src/main/scala/geotrellis/server/ogc/wms/WmsView.scala

@@ -94,16 +94,16 @@ class WmsView(wmsModel: WmsModel, serviceUrl: URL) {
         val re = RasterExtent(wmsReq.boundingBox, wmsReq.width, wmsReq.height)
         wmsModel.getLayer(wmsReq).map { layer =>
           val evalExtent = layer match {
-            case sl @ SimpleOgcLayer(_, _, _, _, _, _) =>
+            case sl @ SimpleOgcLayer(_, _, _, _, _, _, _) =>
               LayerExtent.identity(sl)
-            case MapAlgebraOgcLayer(_, _, _, parameters, expr, _, _) =>
+            case MapAlgebraOgcLayer(_, _, _, parameters, expr, _, _, _) =>
               LayerExtent(IO.pure(expr), IO.pure(parameters), ConcurrentInterpreter.DEFAULT[IO])
           }
 
           val evalHisto = layer match {
-            case sl @ SimpleOgcLayer(_, _, _, _, _, _) =>
+            case sl @ SimpleOgcLayer(_, _, _, _, _, _, _) =>
               LayerHistogram.identity(sl, 512)
-            case MapAlgebraOgcLayer(_, _, _, parameters, expr, _, _) =>
+            case MapAlgebraOgcLayer(_, _, _, parameters, expr, _, _, _) =>
               LayerHistogram(IO.pure(expr), IO.pure(parameters), ConcurrentInterpreter.DEFAULT[IO], 512)
           }
 

ogc-example/src/main/scala/geotrellis/server/ogc/wmts/WmtsView.scala

@@ -60,16 +60,16 @@ class WmtsView(wmtsModel: WmtsModel, serviceUrl: URL) {
           val layerName = wmtsReq.layer
           wmtsModel.getLayer(wmtsReq).map { layer =>
             val evalWmts = layer match {
-              case sl @ SimpleTiledOgcLayer(_, _, _, _, _, _, _) =>
+              case sl @ SimpleTiledOgcLayer(_, _, _, _, _, _, _, _) =>
                 LayerTms.identity(sl)
-              case MapAlgebraTiledOgcLayer(_, _, _, _, parameters, expr, _, _) =>
+              case MapAlgebraTiledOgcLayer(_, _, _, _, parameters, expr, _, _, _) =>
                 LayerTms(IO.pure(expr), IO.pure(parameters), ConcurrentInterpreter.DEFAULT[IO])
             }
 
             val evalHisto = layer match {
-              case sl@SimpleTiledOgcLayer(_, _, _, _, _, _, _) =>
+              case sl@SimpleTiledOgcLayer(_, _, _, _, _, _, _, _) =>
                 LayerHistogram.identity(sl, 512)
-              case MapAlgebraTiledOgcLayer(_, _, _, _, parameters, expr, _, _) =>
+              case MapAlgebraTiledOgcLayer(_, _, _, _, parameters, expr, _, _, _) =>
                 LayerHistogram(IO.pure(expr), IO.pure(parameters), ConcurrentInterpreter.DEFAULT[IO], 512)
             }
 

ogc/src/main/scala/geotrellis/server/ogc/OgcLayer.scala

@@ -22,12 +22,12 @@ import geotrellis.raster._
 import geotrellis.raster.resample._
 import geotrellis.raster.io.geotiff._
 import geotrellis.raster.reproject.ReprojectRasterExtent
+import geotrellis.raster.reproject.Reproject.Options
 import geotrellis.vector.Extent
 import geotrellis.proj4.CRS
 import com.azavea.maml.ast._
 import cats.effect._
 import cats.data.{NonEmptyList => NEL}
-import geotrellis.raster.reproject.Reproject.Options
 
 /**
  * OgcLayer instances are sufficient to produce visual rasters as the end product of 'get map'
@@ -41,6 +41,7 @@ sealed trait OgcLayer {
   def crs: CRS
   def style: Option[OgcStyle]
   def resampleMethod: ResampleMethod
+  def overviewStrategy: OverviewStrategy
 }
 
 sealed trait RasterOgcLayer {
@@ -53,7 +54,8 @@ case class SimpleOgcLayer(
   crs: CRS,
   source: RasterSource,
   style: Option[OgcStyle],
-  resampleMethod: ResampleMethod
+  resampleMethod: ResampleMethod,
+  overviewStrategy: OverviewStrategy
 ) extends OgcLayer with RasterOgcLayer
 
 case class MapAlgebraOgcLayer(
@@ -63,7 +65,8 @@ case class MapAlgebraOgcLayer(
   parameters: Map[String, SimpleOgcLayer],
   algebra: Expression,
   style: Option[OgcStyle],
-  resampleMethod: ResampleMethod
+  resampleMethod: ResampleMethod,
+  overviewStrategy: OverviewStrategy
 ) extends OgcLayer
 
 object SimpleOgcLayer {
@@ -75,7 +78,7 @@ object SimpleOgcLayer {
         logger.trace(s"attempting to retrieve layer $self at extent $extent with $cs ${targetGrid.cols}x${targetGrid.rows}")
         logger.trace(s"Requested extent geojson: ${extent.toGeoJson}")
         val raster: Raster[MultibandTile] = self.source
-          .reprojectToRegion(self.crs, targetGrid.toRasterExtent, self.resampleMethod, AutoHigherResolution)
+          .reprojectToRegion(self.crs, targetGrid.toRasterExtent, self.resampleMethod, self.overviewStrategy)
           .read(extent)
           .getOrElse(throw new Exception(s"Unable to retrieve layer $self at extent $extent $cs"))
         logger.trace(s"Successfully retrieved layer $self at extent $extent with f $cs ${targetGrid.cols}x${targetGrid.rows}")