Merge pull request #3230 from Opetushallitus/tor-2262-ib-kali

TOR-2262: IB-käliuudistus

.github/actions/koski_frontend_test/action.yml

@@ -20,11 +20,8 @@ runs:
       env:
         MOCHA_SHARD_INDEX: ${{inputs.shardIndex}}
         MOCHA_SHARD_TOTAL: ${{inputs.shardTotal}}
-      uses: nick-fields/retry@v3
-      with:
-        max_attempts: 2
-        timeout_minutes: 45
-        command: mvn scalatest:test -Dsuites="fi.oph.koski.mocha.KoskiParallelMochaSpec" --batch-mode
+      run: mvn scalatest:test -Dsuites="fi.oph.koski.mocha.KoskiParallelMochaSpec" --batch-mode
+      shell: bash
 
     - name: Archive test screenshots
       if: ${{ failure() }}

.github/workflows/all_tests.yml

@@ -246,8 +246,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6, 7]
-        shardTotal: [7]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+        shardTotal: [12]
     runs-on: ${{ inputs.runs-on }}
     if: ${{ !contains(github.event.head_commit.message, '[skip-tests]') }}
     needs: [build_koski]

src/main/resources/localization/koski-default-texts.json

@@ -1194,6 +1194,7 @@
   "description:Suoritettavan tutkinnon osan tunnistetiedot": "Suoritettavan tutkinnon osan tunnistetiedot",
   "Jakolinkki luotu": "Jakolinkki luotu",
   "kurssia": "kurssia",
+  "opintopistettä": "opintopistettä",
   "Jakolinkin poistaminen onnistui.": "Jakolinkin poistaminen onnistui.",
   "Muutokset tallennettu": "Muutokset tallennettu",
   "Opiskelu-oikeuksien lukumäärä": "Opiskelu-oikeuksien lukumäärä",
@@ -2761,5 +2762,8 @@
   "raportti-excel-must-tiedoston-etuliite": "muks_suoritustiedot",
   "raportti-excel-kolumni-yhteislaajuusKaikkiTuntia": "Yhteislaajuus (tuntia)",
   "Siirtynyt uusiin tutkinnon perusteisiin": "Siirtynyt uusiin tutkinnon perusteisiin",
-  "description:Onko opiskeluoikeuden päättymisen syy siirtyminen...": "Onko opiskeluoikeuden päättymisen syy siirtyminen tutkinnon uusiin perusteisiin."
+  "description:Onko opiskeluoikeuden päättymisen syy siirtyminen...": "Onko opiskeluoikeuden päättymisen syy siirtyminen tutkinnon uusiin perusteisiin.",
+  "IB-oppiaine": "IB-oppiaine",
+  "Lukion oppiaine": "Lukion oppiaine",
+  "Aineryhmä": "Aineryhmä"
 }

src/main/scala/fi/oph/koski/typemodel/Constraints.scala

@@ -91,6 +91,8 @@ object Constraints {
               )
             case _ => AnyConstraint()
           }
+        case b: BoundaryType =>
+          RefConstraint(className = b.className)
         case _: AnyType =>
           AnyConstraint()
       }
@@ -188,3 +190,9 @@ case class AnyConstraint(
   @EnumValue("any")
   `type`: String = "any",
 ) extends Constraint
+
+case class RefConstraint(
+  @EnumValue("objectRef")
+  `type`: String = "objectRef",
+  className: String
+) extends Constraint

src/main/scala/fi/oph/koski/typemodel/SchemaExport.scala

@@ -1,19 +1,29 @@
 package fi.oph.koski.typemodel
 
 import fi.oph.scalaschema._
-import fi.oph.scalaschema.annotation.DefaultValue
 
 import scala.language.implicitConversions
 
 object SchemaExport {
-  def toTypeDef(schema: Schema, followClassRefs: Boolean = false): Seq[TypeModelWithClassName] =
-    parse(schema, followClassRefs).collect { case s: TypeModelWithClassName => s }
+  case class Context(
+    followClassRefs: Boolean,
+    depth: Int,
+    stopParsingAt: List[String]
+  ) {
+    def next(): Context = copy(depth = depth - 1)
+    def isEnd: Boolean = depth <= 0
+    def isEnd(className: String): Boolean = depth <= 0 || stopParsingAt.contains(className)
+  }
+
+  def toTypeDef(schema: Schema, followClassRefs: Boolean = false, maxDepth: Int = Int.MaxValue, stopParsingAt: List[String] = Nil): Seq[TypeModelWithClassName] =
+    parse(schema, Context(followClassRefs, maxDepth, stopParsingAt)).collect { case s: TypeModelWithClassName => s }
 
-  private def parse(schema: Schema, followClassRefs: Boolean = false): Seq[TypeModel] =
-    Seq(parseSchema(schema)) ++ parseAssociatedSchemas(schema, followClassRefs)
+  private def parse(schema: Schema, ctx: Context): Seq[TypeModel] =
+    Seq(parseSchema(schema, ctx)) ++ parseAssociatedSchemas(schema, ctx)
 
-  private def parseSchema(schema: Schema): TypeModel = {
+  private def parseSchema(schema: Schema, ctx: Context): TypeModel = {
     (schema match {
+      case s: SchemaWithClassName if ctx.isEnd(s.fullClassName) => BoundaryType(s.fullClassName)
       case StringSchema(enumValues) =>
         enumValues match {
           case Some(enumValues) => EnumType(DataTypes.String, enumValues)
@@ -32,24 +42,24 @@ object SchemaExport {
       case DateSchema(_) =>
         DateType() // TODO: Pitää varmaan tukea myös kellonaikoja
       case OptionalSchema(itemSchema) =>
-        OptionalType(parseSchema(itemSchema))
+        OptionalType(parseSchema(itemSchema, ctx.next()))
       case ListSchema(itemSchema) =>
-        ArrayType(parseSchema(itemSchema))
+        ArrayType(parseSchema(itemSchema, ctx.next()))
       case MapSchema(itemSchema) =>
-        RecordType(parseSchema(itemSchema))
+        RecordType(parseSchema(itemSchema, ctx.next()))
       case schema: ClassSchema if schema.readFlattened.isDefined =>
-        parseSchema(schema.asAnyOfSchema)
+        parseSchema(schema.asAnyOfSchema, ctx.next())
       case classSchema: ClassSchema =>
-        parseClassSchema(classSchema)
+        parseClassSchema(classSchema, ctx.next())
       case classRef: ClassRefSchema =>
         ClassRef(classRef.fullClassName)
       case anyOf: AnyOfSchema =>
         UnionType(
           fullClassName = anyOf.fullClassName,
-          anyOf = anyOf.alternatives.map(parseSchema)
+          anyOf = anyOf.alternatives.map(parseSchema(_, ctx.next()))
         )
       case flattenedSchema: FlattenedSchema =>
-        parseSchema(flattenedSchema.property.schema)
+        parseSchema(flattenedSchema.property.schema, ctx.next())
       case _: AnyObjectSchema =>
         AnyObjectType()
       case _: AnyListSchema =>
@@ -59,19 +69,23 @@ object SchemaExport {
     }).withMetadata(schema.metadata)
   }
 
-  private def parseAssociatedSchemas(schema: Schema, followClassRefs: Boolean = false): Seq[TypeModel] = schema match {
-    case classSchema: ClassSchema => classSchema.definitions.flatMap(clss => parse(clss))
-    case classRefSchema: ClassRefSchema if followClassRefs =>
-      println(s"Parse associated schemas: ref ${classRefSchema.fullClassName}")
-      TypeExport.getObjectModels(Class.forName(classRefSchema.fullClassName))
-    case anyOfSchema: AnyOfSchema =>
-      println(s"Parse associated schemas: anyOf ${anyOfSchema.fullClassName}")
-      anyOfSchema.alternatives.flatMap(alt => parseAssociatedSchemas(alt, followClassRefs))
-    case _ => Seq.empty
+  private def parseAssociatedSchemas(schema: Schema, ctx: Context): Seq[TypeModel] = {
+    schema match {
+      case _ if ctx.isEnd => Seq.empty
+      case classSchema: ClassSchema => classSchema.definitions.flatMap(clss => parse(clss, ctx.next()))
+      case classRefSchema: ClassRefSchema if ctx.followClassRefs =>
+        // println(s"Parse associated schemas: ref ${classRefSchema.fullClassName}")
+        TypeExport.getObjectModels(Class.forName(classRefSchema.fullClassName))
+      case anyOfSchema: AnyOfSchema =>
+        // println(s"Parse associated schemas: anyOf ${anyOfSchema.fullClassName}")
+        anyOfSchema.alternatives.flatMap(alt => parseAssociatedSchemas(alt, ctx.next()))
+      case _ => Seq.empty
+    }
   }
 
-  private def parseClassSchema(classSchema: ClassSchema): TypeModel = {
-    val propTypes = classSchema.properties.map(p => parseSchema(p.schema).withMetadata(p.metadata))
+  private def parseClassSchema(classSchema: ClassSchema, ctx: Context): TypeModel = {
+    val nextCtx = ctx.next()
+    val propTypes = classSchema.properties.map(p => parseSchema(p.schema, nextCtx).withMetadata(p.metadata))
     val propKeys = classSchema.properties.map(_.key)
     ObjectType(
       fullClassName = classSchema.fullClassName,

src/main/scala/fi/oph/koski/typemodel/TsFileUpdater.scala

@@ -82,4 +82,10 @@ case class AdditionalExports(
   vapaanSivistystyönKoulutuksenPäätasonSuoritus: VapaanSivistystyönKoulutuksenPäätasonSuoritus,
   suorituskielellinen: Suorituskielellinen,
   maksuttomuustieto: MaksuttomuusTieto,
+  valinnanMahdollisuus: ValinnanMahdollisuus,
+  arvioinniton: Arvioinniton,
+  mahdollisestiArvioinniton: MahdollisestiArvioinniton,
+  valinnaisuus: Valinnaisuus,
+  preIBLukionModuulinSuoritus2019: PreIBLukionModuulinSuoritus2019,
+  preIBLukionModuulinTaiPaikallisenOpintojaksonSuoritus2019: PreIBLukionModuulinTaiPaikallisenOpintojaksonSuoritus2019,
 )

src/main/scala/fi/oph/koski/typemodel/TypeExport.scala

@@ -3,16 +3,16 @@ package fi.oph.koski.typemodel
 import fi.oph.scalaschema.SchemaFactory
 
 object TypeExport {
-  def toTypeDef(clss: Class[_], followClassRefs: Boolean = false): Seq[TypeModelWithClassName] = {
-    val schemaFactory: SchemaFactory = SchemaFactory()
+  val schemaFactory: SchemaFactory = SchemaFactory()
+
+  def toTypeDef(clss: Class[_], followClassRefs: Boolean = false, maxDepth: Int = Int.MaxValue, stopParsingAt: List[String] = Nil): Seq[TypeModelWithClassName] = {
     val schema = schemaFactory.createSchema(clss)
-    SchemaExport.toTypeDef(schema, followClassRefs)
+    SchemaExport.toTypeDef(schema, followClassRefs, maxDepth, stopParsingAt)
   }
 
   def getObjectModels(clss: Class[_]): List[ObjectType] =
     getTypeModel(clss).toList.flatMap(getObjectModels)
 
-
   def getObjectModels(model: TypeModel): List[ObjectType] =
     model match {
       case a: ArrayType => getObjectModels(a.items)

src/main/scala/fi/oph/koski/typemodel/TypeModel.scala

@@ -266,6 +266,10 @@ case class AnyType(
   def `type`: DataType = DataTypes.Any
 }
 
+case class BoundaryType(className: String) extends TypeModel {
+  def `type`: DataType = DataTypes.Object
+}
+
 // Common
 
 case class Limit(n: Double, inclusive: Boolean)

src/main/scala/fi/oph/koski/typemodel/TypeModelServlet.scala

@@ -17,7 +17,11 @@ class TypeModelServlet(implicit val application: KoskiApplication)
 {
   get("/constraints/:schemaClass") {
     val className = s"fi.oph.koski.schema.${params("schemaClass").replaceAll("[^\\w\\däÄöÖ]", "")}"
-    val typeAndSubtypes = TypeExport.toTypeDef(Class.forName(className), followClassRefs = true)
+    val typeAndSubtypes = TypeExport.toTypeDef(
+      clss = Class.forName(className),
+      followClassRefs = true,
+      stopParsingAt = stopSchemaParsingForConstraintsAt,
+    )
     val requestedType = typeAndSubtypes.find(_.fullClassName == className)
     val constraint = requestedType
       .map(t => Constraints.build(t, typeAndSubtypes))
@@ -85,6 +89,11 @@ class TypeModelServlet(implicit val application: KoskiApplication)
     .anyOf
     .collect { case ClassRef(c) => c }
     .flatMap(OpiskeluoikeusClass.apply)
+
+  // Lisää tähän sellaiset luokat, jotka aiheuttavat valtavien tietomallivastausten luonnin
+  private lazy val stopSchemaParsingForConstraintsAt: List[String] = List(
+    "fi.oph.koski.schema.OsaamisenTunnustaminen",
+  )
 }
 
 case class GroupedKoodistot(

web/app/appstate/constraints.tsx

@@ -1,5 +1,4 @@
 import * as E from 'fp-ts/Either'
-import * as C from '../util/constraints'
 import { pipe } from 'fp-ts/lib/function'
 import React, {
   useCallback,
@@ -9,9 +8,10 @@ import React, {
   useState
 } from 'react'
 import { Constraint } from '../types/fi/oph/koski/typemodel/Constraint'
+import * as C from '../util/constraints'
 import { fetchConstraint } from '../util/koskiApi'
-import { ClassOf, ObjWithClass, schemaClassName } from '../util/types'
 import { PropsWithOnlyChildren } from '../util/react'
+import { ClassOf, ObjWithClass, schemaClassName } from '../util/types'
 
 /*
  * CONSTRAINTS
@@ -54,7 +54,10 @@ import { PropsWithOnlyChildren } from '../util/react'
  * @param className skeemaluokan nimi pitkässä tai lyhyessä muodossa (esim. "fi.oph.koski.schema.Vahvistus" tai pelkkä "Vahvistus")
  * @returns Constraint jos luokka löytyi ja lataaminen onnistui, null jos haku kesken tai tietojen lataaminen epäonnistui.
  */
-export const useSchema = (className?: string | null): Constraint | null => {
+export const useSchema = (
+  className?: string | null,
+  shallow: boolean = false
+): Constraint | null => {
   const { constraints, loadConstraint } = useContext(ConstraintsContext)
   const schemaClass = useMemo(
     () => (className && schemaClassName(className)) || className,
@@ -84,10 +87,7 @@ export const useChildSchema = (
   path: string
 ): Constraint | null => {
   const c = useSchema(className)
-  return useMemo(
-    () => pipe(c, C.asList, C.path(path), (cs) => cs?.[0] || null),
-    [c, path]
-  )
+  return useMemo(() => pipe(c, C.asList, C.path(path), C.join), [c, path])
 }
 
 /**