Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for the x-ci-accept-failures custom field #62

Merged
merged 3 commits into from
Jan 5, 2021
Merged

Add support for the x-ci-accept-failures custom field #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
eb73890
Optimization: combine an unused obuilder step with the next one
kit-ty-kate Dec 31, 2020
e0b27d1
Add support for the x-ci-accept-failures custom field
kit-ty-kate Dec 31, 2020
6006310
Give distribution name instead of guessing using opam
kit-ty-kate Jan 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 31 additions & 33 deletions lib/opam_build.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ let download_cache = "opam-archives"
let cache = [ Obuilder_spec.Cache.v download_cache ~target:"/home/opam/.opam/download-cache" ]
let network = ["host"]

let opam_install ~upgrade_opam ~pin ~with_tests ~pkg =
let opam_install ~variant ~upgrade_opam ~pin ~with_tests ~pkg =
let pkg = OpamPackage.to_string pkg in
let open Obuilder_spec in
let pin =
Expand All @@ -16,42 +16,40 @@ let opam_install ~upgrade_opam ~pin ~with_tests ~pkg =
[]
in
pin @ [
run ~cache "opam remove -y %s" pkg;
(* TODO: Replace by two calls to opam install + opam install -t using the OPAMDROPINSTALLEDPACKAGES feature *)
if upgrade_opam then
run ~cache ~network "opam install -y%s %s" (if with_tests then "t" else "") pkg
else
run ~cache ~network "opam depext -uivy%s %s" (if with_tests then "t" else "") pkg
run ~cache ~network {|
opam remove -y %s && opam %s%s %s
res=$?
test "$res" = 0 && exit 0
test "$res" != 31 && exit 1
export OPAMCLI=2.0
build_dir=$(opam var prefix)/.opam-switch/build
failed=$(ls "$build_dir")
for pkg in $failed; do
if opam show -f x-ci-accept-failures: "$pkg" | grep -qF "\"%s\""; then
echo "A package failed and has been disabled for CI using the 'x-ci-accept-failures' field."
fi
done
exit 1
|}
pkg
(if upgrade_opam then "install -y" else "depext -uivy")
(if with_tests then "t" else "")
pkg
(Variant.distribution variant)
]

let setup_repository ~upgrade_opam ~variant =
let setup_repository ~upgrade_opam =
let open Obuilder_spec in
let variant = Variant.docker_tag variant in
let opam_extras =
if Astring.String.is_suffix ~affix:"-ocaml-4.07" variant ||
Astring.String.is_suffix ~affix:"-ocaml-4.06" variant ||
Astring.String.is_suffix ~affix:"-ocaml-4.05" variant ||
Astring.String.is_suffix ~affix:"-ocaml-4.04" variant ||
Astring.String.is_suffix ~affix:"-ocaml-4.03" variant ||
Astring.String.is_suffix ~affix:"-ocaml-4.02" variant then
[ run ~cache ~network "opam install -y ocaml-secondary-compiler" ] (* Speed up builds for dune >= 2.0 *)
else
[]
in
let distro_extras =
if Astring.String.is_prefix ~affix:"fedora" variant then
[ run ~network "sudo dnf install -y findutils" ] (* (we need xargs) *)
else
[]
in
(if upgrade_opam then [
run "sudo mv /usr/bin/opam-2.1 /usr/bin/opam";
env "OPAMDEPEXTYES" "1"] else []) @
env "OPAMDOWNLOADJOBS" "1" :: (* Try to avoid github spam detection *)
env "OPAMERRLOGLEN" "0" :: (* Show the whole log if it fails *)
env "OPAMSOLVERTIMEOUT" "500" :: (* Increase timeout. Poor mccs is doing its best *)
env "OPAMPRECISETRACKING" "1" :: (* Mitigate /~https://github.com/ocaml/opam/issues/3997 *)
user ~uid:1000 ~gid:1000 :: distro_extras @ opam_extras @ [
[
user ~uid:1000 ~gid:1000;
copy ["."] ~dst:"/src/";
run "opam repository set-url --strict default file:///src";
]
Expand All @@ -66,28 +64,28 @@ let spec ~upgrade_opam ~base ~variant ~revdep ~with_tests ~pkg =
let open Obuilder_spec in
let revdep = match revdep with
| None -> []
| Some revdep -> opam_install ~upgrade_opam ~pin:false ~with_tests:false ~pkg:revdep
| Some revdep -> opam_install ~variant ~upgrade_opam ~pin:false ~with_tests:false ~pkg:revdep
and tests = match with_tests, revdep with
| true, None -> opam_install ~upgrade_opam ~pin:false ~with_tests:true ~pkg
| true, Some revdep -> opam_install ~upgrade_opam ~pin:false ~with_tests:true ~pkg:revdep
| true, None -> opam_install ~variant ~upgrade_opam ~pin:false ~with_tests:true ~pkg
| true, Some revdep -> opam_install ~variant ~upgrade_opam ~pin:false ~with_tests:true ~pkg:revdep
| false, _ -> []
in
{ from = base;
ops =
set_personality ~variant
@ setup_repository ~upgrade_opam ~variant
@ opam_install ~upgrade_opam ~pin:true ~with_tests:false ~pkg
@ setup_repository ~upgrade_opam
@ opam_install ~variant ~upgrade_opam ~pin:true ~with_tests:false ~pkg
@ revdep
@ tests
}

let revdeps ~with_tests ~base ~variant ~pkg =
let revdeps ~with_tests ~base ~variant:_ ~pkg =
let open Obuilder_spec in
let pkg = Filename.quote (OpamPackage.to_string pkg) in
let with_tests = if with_tests then " --with-test" else "" in
{ from = base;
ops =
setup_repository ~upgrade_opam:false ~variant
setup_repository ~upgrade_opam:false
@ [
run "echo '@@@OUTPUT' && \
opam list -s --color=never --depends-on %s --coinstallable-with %s --installable --all-versions --recursive --depopts%s && \
Expand Down
17 changes: 13 additions & 4 deletions lib/variant.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,22 @@ type arch = Ocaml_version.arch
let arch_to_yojson arch = `String (Ocaml_version.string_of_arch arch)

type t = {
docker_tag : string;
distribution : string;
ocaml_version : string;
ocaml_variant : string option;
arch : arch;
} [@@deriving to_yojson]

let arch t = t.arch
let docker_tag t = t.docker_tag
let docker_tag t =
let variant = match t.ocaml_variant with
| None -> ""
| Some variant -> "-"^variant
in
t.distribution^"-ocaml-"^t.ocaml_version^variant
let distribution t = t.distribution

let pp f { docker_tag; arch } = Fmt.pf f "%s/%s" docker_tag (Ocaml_version.string_of_arch arch)
let pp f t = Fmt.pf f "%s/%s" (docker_tag t) (Ocaml_version.string_of_arch t.arch)

let v ~arch docker_tag = { arch; docker_tag }
let v ~arch ~distro ~compiler:(ocaml_version, ocaml_variant) =
{ arch; distribution = distro; ocaml_version; ocaml_variant }
8 changes: 4 additions & 4 deletions service/pipeline.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ let build_with_cluster ~ocluster ~analysis ~master source =
let v = Ocaml_version.with_just_major_and_minor v in
let revdeps = Ocaml_version.equal v default_compiler in (* TODO: Remove this when the cluster is ready *)
let v = Ocaml_version.to_string v in
let variant = Variant.v ~arch:`X86_64 @@ master_distro^"-ocaml-"^v in
let variant = Variant.v ~arch:`X86_64 ~distro:master_distro ~compiler:(v, None) in
build ~revdeps v variant
)
end
Expand All @@ -213,22 +213,22 @@ let build_with_cluster ~ocluster ~analysis ~master source =
acc
else
let distro = Dockerfile_distro.tag_of_distro distro in
let variant = Variant.v ~arch:`X86_64 @@ distro^"-ocaml-"^default_compiler in
let variant = Variant.v ~arch:`X86_64 ~distro ~compiler:(default_compiler, None) in
build ~revdeps:false distro variant :: acc
) []
end
and+ extras =
let master_distro = Dockerfile_distro.tag_of_distro master_distro in
let default_compiler = Ocaml_version.to_string default_compiler in
let flambda = Variant.v ~arch:`X86_64 (master_distro^"-ocaml-"^default_compiler^"-flambda") in
let flambda = Variant.v ~arch:`X86_64 ~distro:master_distro ~compiler:(default_compiler, Some "flambda") in
Current.list_seq (
build ~upgrade_opam:true ~revdeps:false "flambda" flambda ::
List.fold_left (fun acc arch ->
if arch = `X86_64 then
acc
else
let label = Ocaml_version.to_opam_arch arch in
let variant = Variant.v ~arch (master_distro^"-ocaml-"^default_compiler) in
let variant = Variant.v ~arch ~distro:master_distro ~compiler:(default_compiler, None) in
build ~upgrade_opam:true ~revdeps:false label variant :: acc
) [] Ocaml_version.arches
)
Expand Down