Merge pull request #223 from dra27/live-into-master

Merge live branch back into master

.dockerignore

@@ -0,0 +1,2 @@
+_build
+_opam

.gitignore

@@ -1,4 +1,5 @@
 _build
+_opam
 .merlin
 *.install
 www

Dockerfile

@@ -1,12 +1,11 @@
-# syntax=docker/dockerfile:1
-FROM ocaml/opam:alpine-3.14-ocaml-4.10 as build-opam2web
-RUN sudo apk add g++
-RUN git clone /~https://github.com/ocaml/opam2web --depth 1 /home/opam/opam2web
+# syntax=docker/dockerfile:1.3
+FROM ocaml/opam:alpine-3.15-ocaml-4.14 as build-opam2web
+RUN sudo apk add g++ gmp-dev
+RUN git clone /~https://github.com/ocaml/opam2web.git --depth 1 /home/opam/opam2web
 WORKDIR /home/opam/opam2web
 ENV OCAMLRUNPARAM b
 RUN sudo mkdir -p /opt/opam2web && sudo chown opam:opam /opt/opam2web
-RUN sudo mv /usr/bin/opam-2.1 /usr/bin/opam
-RUN opam repo set-url default https://opam-dev.ocaml.org/
+RUN opam repo set-url default git+/~https://github.com/ocaml/opam-repository.git#${OPAM_REPO_GIT_SHA}
 RUN opam install . --destdir /opt/opam2web
 RUN cp -r content /opt/opam2web/share/opam2web/
 RUN rm -rf /opt/opam2web/share/opam2web/lib
@@ -17,7 +16,8 @@ RUN git clone /~https://github.com/ocaml/opam.wiki.git --depth 1 -b old_wiki /opt/
 RUN git clone /~https://github.com/ocaml/opam --depth 1 -b 1.2 /tmp/opam-1.2 \
     && mv /tmp/opam-1.2/doc/pages /opt/opam2web/share/opam2web/content/doc/1.2 \
     && rm -rf /tmp/opam-1.2
-FROM ocaml/opam:alpine-3.14-ocaml-4.10 as build-opam-doc
+
+FROM ocaml/opam:alpine-3.15-ocaml-4.14 as build-opam-doc
 RUN sudo apk add cgit groff
 RUN sudo mkdir -p /usr/local/bin \
     && echo -e '#!/bin/sh -e\n\
@@ -40,11 +40,26 @@ RUN sudo mkdir -p /opt/opam/doc && sudo chown -R opam:opam /opt/opam
 RUN cp -r doc/html /opt/opam/doc/api
 RUN cp -r doc/man-html /opt/opam/doc/man
 RUN cp -r doc/pages/* /opt/opam/doc/
-FROM alpine:3.14 as opam2web
-RUN apk add git curl rsync libstdc++
+
+FROM --platform=linux/amd64 ocaml/opam:archive as opam-archive
+FROM ocaml/opam.ocaml.org-legacy as opam-legacy
+FROM alpine:3.15 as opam2web
+RUN apk add --update git curl rsync libstdc++ rdfind
+COPY --from=opam-legacy . /www
 COPY --from=build-opam2web /opt/opam2web /usr/local
 COPY --from=build-opam-doc /usr/bin/opam /usr/local/bin/opam
 COPY --from=build-opam-doc /opt/opam/doc /usr/local/share/opam2web/content/doc
+RUN --mount=type=bind,target=/cache,from=opam-archive rsync -aH /cache/cache/ /www/cache/
+COPY ext/key/opam-dev-team.pgp /www/opam-dev-pubkey.pgp
 ADD bin/opam-web.sh /usr/local/bin
-VOLUME ["/persist"]
-VOLUME ["/www"]
+ARG DOMAIN=opam.ocaml.org
+ARG OPAM_REPO_GIT_SHA=master
+ARG BLOG_GIT_SHA=master
+RUN echo ${OPAM_REPO_GIT_SHA} >> /www/opam_git_sha
+RUN echo ${BLOG_GIT_SHA} >> /www/blog_git_sha
+RUN /usr/local/bin/opam-web.sh ${DOMAIN} ${OPAM_REPO_GIT_SHA} ${BLOG_GIT_SHA}
+
+FROM caddy:2.5.2-alpine
+WORKDIR /srv
+COPY --from=opam2web /www /usr/share/caddy
+ENTRYPOINT ["caddy", "file-server"]

README.md

@@ -4,17 +4,24 @@ opam2web
 A tool to generate a website from an OPAM universe
 
 This utility creates a static website from an OPAM universe, listing all
-available packages and their details. A homepage and OPAM documentation is 
+available packages and their details. A homepage and OPAM documentation is
 included as well.
 
 The latest release of opam2web is available via
 [opam](http://opam.ocaml.org). To install, simply run:
+
 ```bash
 opam install opam2web
 ```
 
 ### Prerequisties
 
+Optionally create a local switch for the project:
+
+``` bash
+opam switch create . ocaml.4.14.0 --with-test
+```
+
 - re [github.com/ocaml/ocaml-re](/~https://github.com/ocaml/ocaml-re)
 - uri [github.com/avsm/ocaml-uri](/~https://github.com/avsm/ocaml-uri)
 - opam [github.com/ocaml/opam](/~https://github.com/ocaml/opam)
@@ -23,26 +30,37 @@ opam install opam2web
 - cmdliner [erratique.ch/software/cmdliner](http://erratique.ch/software/cmdliner)
 - js_of_ocaml [ocsigen.org/js_of_ocaml](http://ocsigen.org/js_of_ocaml/)
 
-If you have opam installed:
+### Build
+
+`dune exec -- opam2web` will compile and run the utility directly, alternatively you can run:
+
 ```bash
-opam install re uri opam-lib opamfu cow cmdliner js_of_ocaml
+dune build @all
 ```
 
-### Build
+and the binary will be located in `_build/install/default/bin/opam2web` after compilation.
 
-To build the `opam2web` utility, enter:
-```bash
-make
+At this point you'll either want to try a `docker` build or running the `opam2web` cli locally.
+
+### Docker
+
+The website generation for opam.ocaml.org uses a combination of `docker` and `ocurrent-deployer` to rebuild the site. 
+To replicate the docker image run:
+
+``` bash
+DOCKER_BUILDKIT=1 docker build -t opam2web .
 ```
-The binary will be located in src/_build/opam2web.native after compilation.
 
-To generate the static website corresponding to the `default` remote in the
-local OPAM installation, enter:
-```bash
-make -C src run
+which uses the local Dockerfile and creates an image called `opam2web`. Note this image is rather large at over 18Gb and takes a while to build.
+Once built it can be run as `docker run -p 127.0.0.1:8080:80/tcp opam2web --root /usr/share/caddy` and viewable on http://localhost:8080.
+
+To run the image produced by deploy.ci.ocaml.org run (note the image is multi-arch with x86 and ARM64 support):
+
+``` bash
+docker run -p 127.0.0.1:8080:80/tcp --name opam2web ocurrent/opam.ocaml.org:live --root /usr/share/caddy
 ```
 
-### Usage
+### CLI Usage
 
 ```bash
 opam2web [options]* [repositories]*
@@ -83,6 +101,7 @@ Some available predicates are:
 - `pkg:*`
 
 For complete command-line configuration options, run
+
 ```bash
 opam2web --help
 ```
@@ -96,7 +115,19 @@ will generate the HTML files corresponding to the repository located in
 `~/myrepo` and the remote named `default` in the local OPAM installation.
 Resulting files will be located in the `website` directory.
 
-### TODO
 
-- More complex news system (one page per news, Markdown...)
-- More complex statistics (graphics over time...)
+## Deployment
+
+The deployment uses [ocurrent-deployer][] to watch particular branches on this repository, plus the default branches of [opam-repository][] and [platform-blog][]. 
+When any of them change, it calculates if it needs to rebuild the docker image. A site rebuild is therefore triggered by any merge to opam-repository.
+
+ * `live` branch is deployed on [opam.ocaml.org][]
+ * `staging` branch is deployed on [staging.opam.ocaml.org][]
+
+The deployer service is available at https://deploy.ci.ocaml.org/?repo=ocaml-opam/opam2web and the code for the ocurrent pipeline is in [ocurrent-deployer]().
+
+[ocurrent-deployer]: /~https://github.com/ocurrent/ocurrent-deployer
+[opam-repository]: /~https://github.com/ocaml/opam-repository
+[platform-blog]: /~https://github.com/ocaml/platform-blog
+[opam.ocaml.org]: https://opam.ocaml.org
+[staging.opam.ocaml.org]: https://staging.opam.ocaml.org

bin/opam-web.sh

@@ -2,38 +2,26 @@
 
 set -uex
 
-BASEURL=$1 && shift && [ $# -eq 0 ] || { echo "Usage: $0 BASEURL" && exit 2; }
-
-# /persist may contain:
-#         /opam-repository (bare git clone)
-#         /platform-blog (bare git clone)
-#         /legacy (subdirectories copied verbatim)
-#         /repo-cache (as generated by 'opam admin', 'cache/' and 'archives/' subdirs)
-#         /logs-cache (cache of parsed logs used by opam2web)
-
-cd /persist
-# clone/update opam repo
-if [ -d opam-repository ]; then
-  git -C opam-repository fetch /~https://github.com/ocaml/opam-repository +master:master
-else
-  git clone /~https://github.com/ocaml/opam-repository opam-repository --bare --single-branch --branch master
+if [[ $# -eq 4 ]] ; then
+    echo 'Usage: $0 BASEURL OPAM_REPO_GIT_SHA BLOG_GIT_SHA'
+    exit 2
 fi
 
-# clone/update opam blog
-if [ -d blog ]; then
-  git -C blog fetch /~https://github.com/ocaml/platform-blog +master:master
-else
-  git clone /~https://github.com/ocaml/platform-blog blog --bare --single-branch --branch master
-fi
+BASEURL=$1
+OPAM_REPO_GIT_SHA=$2
+BLOG_GIT_SHA=$3
 
 cd /www
-# clone/update opam-repo to /www
-if [ -d .git ]; then
-  git fetch /persist/opam-repository
-  if ! git reset FETCH_HEAD --hard; then git clean -ffdx; git reset FETCH_HEAD --hard; fi
-else
-  git clone -s /persist/opam-repository .
-fi
+# Checkout a specific commit as supplied by ocurrent-deployer pipeline.
+git clone /~https://github.com/ocaml/opam-repository.git --single-branch --branch master opam-repository &&
+    cd opam-repository &&
+    git checkout ${OPAM_REPO_GIT_SHA} &&
+    cd ..
+
+mv opam-repository/* .
+mv opam-repository/.git .
+rm -rf opam-repository
+
 # Overwrite 'repo' file, and dispatch all non-standard versions
 cat <<EOF >repo
 opam-version: "2.0"
@@ -45,27 +33,21 @@ redirect: [
   "https://${BASEURL}/1.2.2" { opam-version < "2.0~" }
 ]
 EOF
-mkdir -p /persist/repo-cache/archives
-mkdir -p /persist/repo-cache/cache
-mkdir -p archives
-opam admin cache -n --link=archives /persist/repo-cache/cache
-rsync -a /persist/repo-cache/cache .
-echo 'archive-mirrors: "cache"' >> repo
-if [ -d /persist/legacy ]; then rsync -a /persist/legacy .; fi
+opam admin cache --link=archives ./cache
 opam admin index --minimal-urls-txt
 
 cp -r /usr/local/share/opam2web/content /tmp/
-git clone -s /persist/blog /tmp/content/blog
+git clone /~https://github.com/ocaml/platform-blog --single-branch --branch master /tmp/content/blog &&
+    cd /tmp/content/blog &&
+    git checkout ${BLOG_GIT_SHA} &&
+    cd -
 
 rm -rf /www/ext
 mkdir -p /www/ext
 cp -r -L /usr/local/share/opam2web/css /www/ext
 cp -r -L /usr/local/share/opam2web/img /www/ext
 cp -r -L /usr/local/share/opam2web/js /www/ext
 
-mkdir -p /persist/logs-cache ~/.cache
-ln -sf /persist/logs-cache ~/.cache/opam2web2
-
 if [ -r /logs/access.log ]; then
     STATS_ARG="--statistics=/logs/access.log"
 else

bin/opam2web.ml

@@ -250,16 +250,16 @@ let default_cmd =
            ~docv:"REPOSITORY"
            ~doc:"Directories containing the repositories to consider")
   in
-  Term.(pure build $ log_files $ out_dir $ content_dir
-          $ repositories_arg $ root_uri $ blog_source_uri),
-  Term.info "opam2web" ~version ~doc ~man
+  let info = Cmd.info "opam2web" ~version ~doc ~man in
+  let term = Term.(const build $ log_files $ out_dir $ content_dir
+                     $ repositories_arg $ root_uri $ blog_source_uri) in
+  Cmd.v info term
+
 
 let () =
   OpamFormatConfig.init ();
   OpamStd.Config.init ();
   OpamRepositoryConfig.init ();
   OpamSolverConfig.init ();
   OpamStateConfig.init ();
-  match Term.eval default_cmd with
-  | `Error _ -> exit 1
-  | _ -> exit 0
+  exit @@ Cmd.eval default_cmd

bin/restart.sh

@@ -7,9 +7,6 @@ if [ $1 = "" ]; then
   exit 1
 fi
 
-tar -C /var/lib/docker/volumes/opam-website/_data/ -cf - . | docker image import - opam-website-data
-DOCKER_BUILDKIT=1 docker build -t opam-website-live -f Dockerfile.image .
 docker kill opam-website-live || true
-docker rm opam-website-live || true
 docker run -d -p 80:80 -p 443:443 -v caddy-data:/data --name opam-website-live opam-website-live caddy file-server -domain $1
 docker system prune -f

src/o2wBlog.ml

@@ -264,7 +264,7 @@ let make_redirect ~root entries =
   | [] -> Html.p (Html.string "No blog pages.")
   | first_entry::_ ->
       let blog_uri =
-        Uri.(resolve "http" root (of_string "blog/"))
+        Uri.(resolve "http" root (of_string "/blog/"))
       in
       let post_uri =
         Uri.(resolve "http" blog_uri (of_string (first_entry.blog_name^"/")))