Skip to content

Commit

Permalink
Use 7zip as archiver and merge tars instead of extracting
Browse files Browse the repository at this point in the history
  • Loading branch information
bostrot committed May 16, 2024
1 parent 6af3de7 commit e7d8aff
Show file tree
Hide file tree
Showing 9 changed files with 163 additions and 45 deletions.
Binary file added 7zip/7za.dll
Binary file not shown.
Binary file added 7zip/7za.exe
Binary file not shown.
Binary file added 7zip/7zxa.dll
Binary file not shown.
78 changes: 78 additions & 0 deletions lib/api/archive.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import 'dart:io';

/// API for 7-Zip archive operations
class ArchiveApi {
static const _exe = './7zip/7za.exe';

/// Get current path
static Future<String> get currentPath async {
// Get current path
try {
final result = await Process.run("cmd", ["/c", "cd"], runInShell: true);
return result.stdout.toString();
} catch (e) {
throw Exception('Failed to get current path: $e');
}
}

/// Extracts the archive at [archivePath] to [destinationPath]
static Future<void> extract(
String archivePath, String destinationPath) async {
// 7zr.exe x layer2.tar.gz -olayer
// Extract archive
try {
await Process.run(_exe, ['x', archivePath, '-o$destinationPath']);
} catch (e) {
throw Exception('Failed to extract archive: $e');
}
}

/// Merge the archives at [archivePaths] into [destinationPath]
static Future<void> merge(
List<String> archivePaths, String destinationPath) async {
// Merge archives
try {
// remove trailing zeros from the files
final outputFile = File(destinationPath);
for (var i = 0; i < archivePaths.length; i++) {
final fileName = archivePaths[i];
// Read file as byte stream
final file = File(fileName);
final bytes = await file.readAsBytes();
final length = bytes.length;

// Last layer
if (i == archivePaths.length - 1) {
await outputFile.writeAsBytes(bytes);
break;
}

// Remove trailing zeros
int lastBytePos = 0;
for (var i = length - 1; i >= 0; i--) {
if (bytes[i] != 0) {
lastBytePos = i;
break;
}
}

// Write to new file
await outputFile.writeAsBytes(bytes.sublist(0, lastBytePos + 1));
}
} catch (e) {
throw Exception('Failed to merge archives: $e');
}
}

/// Compress the tar archive at [filePath] to [destinationPath]
static Future<void> compress(String filePath, String destinationPath) async {
// 7zr.exe a -tgzip full_image.tar.gz full_image.tar
// Compress tar archive
try {
// 7zr.exe a -ttar combined_image.tar merged\*
await Process.run(_exe, ['a', '-tgzip', destinationPath, filePath]);
} catch (e) {
throw Exception('Failed to compress tar archive: $e');
}
}
}
39 changes: 13 additions & 26 deletions lib/api/docker_images.dart
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,11 @@
import 'dart:convert';
import 'dart:io';
import 'package:archive/archive.dart';
import 'package:archive/archive_io.dart';
import 'package:chunked_downloader/chunked_downloader.dart';
import 'package:dio/dio.dart';
import 'package:flutter/foundation.dart';
import 'package:localization/localization.dart';
import 'package:wsl2distromanager/api/archive.dart';
import 'package:wsl2distromanager/api/safe_paths.dart';
import 'package:wsl2distromanager/components/helpers.dart';
import 'package:wsl2distromanager/components/logging.dart';
Expand Down Expand Up @@ -433,12 +432,13 @@ class DockerImage {
// Write the compressed tar file to disk.
int retry = 0;

String outArchive = SafePath(distroPath).file('$imageName.tar.gz');
final parentPath = SafePath(tmpImagePath);
String outTar = parentPath.file('$imageName.tar');
String outTarGz = SafePath(distroPath).file('$imageName.tar.gz');
while (retry < 2) {
try {
Archive archive = Archive();

// More than one layer
List<String> paths = [];
if (layers != 1) {
for (var i = 0; i < layers; i++) {
// Read archives layers
Expand All @@ -448,34 +448,21 @@ class DockerImage {
// progress(i, layers, -1, -1);
Notify.message('Extracting layer $i of $layers');

// In memory
final tarfile = GZipDecoder().decodeBytes(
File(SafePath(tmpImagePath).file('layer_$i.tar.gz'))
.readAsBytesSync());
final subArchive = TarDecoder().decodeBytes(tarfile);

// Add files to archive
for (final file in subArchive) {
archive.addFile(file);
if (kDebugMode && !file.name.contains('/')) {
if (kDebugMode) {
print('Adding root file ${file.name}');
}
}
}
// Extract layer
final layerTarGz = parentPath.file('layer_$i.tar.gz');
await ArchiveApi.extract(layerTarGz, parentPath.path);
paths.add(parentPath.file('layer_$i.tar'));
}

// Archive as tar then gzip to disk
final tarfile = TarEncoder().encode(archive);
final gzData = GZipEncoder().encode(tarfile);
final fp = File(outArchive);
await ArchiveApi.merge(paths, outTar);
await ArchiveApi.compress(outTar, outTarGz);

Notify.message('writingtodisk-text'.i18n());
fp.writeAsBytesSync(gzData!);
} else if (layers == 1) {
// Just copy the file
File(SafePath(tmpImagePath).file('layer_0.tar.gz'))
.copySync(outArchive);
.copySync(outTarGz);
}

retry = 2;
Expand All @@ -495,7 +482,7 @@ class DockerImage {
Notify.message('creatinginstance-text'.i18n());

// Check if tar file is created
if (!File(outArchive).existsSync()) {
if (!File(outTarGz).existsSync()) {
throw Exception('Tar file is not created');
}
// Wait for tar file to be created
Expand Down
33 changes: 18 additions & 15 deletions lib/dialogs/create_dialog.dart
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,18 @@ createDialog() {
);
}

progressFn(current, total, currentStep, totalStep) {
if (currentStep != -1) {
String progressInMB = (currentStep / 1024 / 1024).toStringAsFixed(2);
// String totalInMB = (total / 1024 / 1024).toStringAsFixed(2);
String percentage = (currentStep / totalStep * 100).toStringAsFixed(0);
Notify.message('${'downloading-text'.i18n()}'
' Layer ${current + 1}/$total: $percentage% ($progressInMB MB)');
} else {
Notify.message('extractinglayers-text'.i18n(['$current', '$total']));
}
}

Future<void> createInstance(
TextEditingController nameController,
TextEditingController locationController,
Expand Down Expand Up @@ -117,21 +129,12 @@ Future<void> createInstance(
// Download image
Notify.message('${'downloading-text'.i18n()}...');
var docker = DockerImage()..distroName = distroName;
await docker.getRootfs(name, image, tag: tag,
progress: (current, total, currentStep, totalStep) {
if (currentStep != -1) {
String progressInMB =
(currentStep / 1024 / 1024).toStringAsFixed(2);
// String totalInMB = (total / 1024 / 1024).toStringAsFixed(2);
String percentage =
(currentStep / totalStep * 100).toStringAsFixed(0);
Notify.message('${'downloading-text'.i18n()}'
' Layer ${current + 1}/$total: $percentage% ($progressInMB MB)');
} else {
Notify.message(
'extractinglayers-text'.i18n(['$current', '$total']));
}
});
try {
await docker.getRootfs(name, image, tag: tag, progress: progressFn);
} catch (e) {
Notify.message('error-text'.i18n());
return;
}
Notify.message('downloaded-text'.i18n());
// Set distropath with distroName
distroName = DockerImage().filename(image, tag);
Expand Down
4 changes: 2 additions & 2 deletions pubspec.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ packages:
dependency: "direct main"
description:
name: archive
sha256: "0763b45fa9294197a2885c8567927e2830ade852e5c896fd4ab7e0e348d0f373"
sha256: ecf4273855368121b1caed0d10d4513c7241dfc813f7d3c8933b36622ae9b265
url: "https://pub.dev"
source: hosted
version: "3.5.0"
version: "3.5.1"
args:
dependency: transitive
description:
Expand Down
2 changes: 1 addition & 1 deletion pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ dependencies:
sdk: flutter
flutter_localizations:
sdk: flutter
archive: ^3.3.7
archive: ^3.5.1
async: ^2.11.0
chunked_downloader: ^0.0.2
desktop_window: ^0.4.0
Expand Down
52 changes: 51 additions & 1 deletion test/dockerimages_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -192,5 +192,55 @@ void main() {
// Verify that the file exists and has > 2MB
expect(await file.exists(), true);
expect(await file.length(), greaterThan(2 * 1024 * 1024));
}, timeout: const Timeout(Duration(minutes: 2)));
}, timeout: const Timeout(Duration(minutes: 10)));

// Test almalinux latest
test('Create instance test almalinux:latest', () async {
TextEditingController nameController = TextEditingController(text: 'test');
TextEditingController locationController = TextEditingController(text: '');
TextEditingController autoSuggestBox =
TextEditingController(text: 'dockerhub:almalinux:latest');

final file =
File('C:/WSL2-Distros/distros/library_almalinux_latest.tar.gz');
if (await file.exists()) {
await file.delete();
}

// Delete the instance
await WSLApi().remove('test');

// Test build context
await createInstance(
nameController,
locationController,
WSLApi(),
autoSuggestBox,
TextEditingController(text: ''),
);

// Verify that the file exists and has > 2MB
expect(await file.exists(), true);
expect(await file.length(), greaterThan(2 * 1024 * 1024));
expect(await isInstance('test'), true);

// Delete the instance
await WSLApi().remove('test');

expect(await isInstance('test'), false);

// Test creating it without re-downloading the rootfs
await createInstance(
nameController,
locationController,
WSLApi(),
autoSuggestBox,
TextEditingController(text: ''),
);

expect(await isInstance('test'), true);

// Delete the instance
await WSLApi().remove('test');
}, timeout: const Timeout(Duration(minutes: 10)));
}

0 comments on commit e7d8aff

Please sign in to comment.