From 44a56935294aebc06b88d857be72832c41d636c0 Mon Sep 17 00:00:00 2001
From: Jan-Hendrik Willms <tleilax+github@gmail.com>
Date: Sat, 14 Jul 2018 20:10:24 +0200
Subject: [PATCH] wip
---
convert.php | 10 ++++--
includes/config.php.dist | 12 +++++--
includes/functions.php | 40 ++++++++++++++++++++++
src/GitLab.php | 7 +++-
src/Migration.php | 10 +++---
src/Trac.php | 6 +++-
steps/add-comments.php | 44 ++++++++++++++++++++++++
steps/clone-svn-repository.php | 34 ++++++++++++++++++
steps/convert-tickets-to-issues.php | 18 +++++++++-
steps/push-repository.php | 7 ++++
steps/rewrite-history.php | 53 +++++++++++++++++++++++++++++
11 files changed, 227 insertions(+), 14 deletions(-)
create mode 100644 includes/functions.php
create mode 100644 steps/add-comments.php
create mode 100644 steps/clone-svn-repository.php
create mode 100644 steps/push-repository.php
create mode 100644 steps/rewrite-history.php
diff --git a/convert.php b/convert.php
index 5211ca2..cd14f5f 100755
--- a/convert.php
+++ b/convert.php
@@ -1,14 +1,18 @@
#!/usr/bin/env php
<?php
require_once __DIR__ . '/vendor/autoload.php';
+require_once __DIR__ . '/includes/functions.php';
$config = require __DIR__ . '/includes/config.php';
$steps = [
- 1 => __DIR__ . '/steps/convert-tickets-to-issues.php',
-// 2 => '',
+ 'issues' => __DIR__ . '/steps/convert-tickets-to-issues.php',
+ 'commits' => __DIR__ . '/steps/clone-svn-repository.php',
+ 'history' => __DIR__ . '/steps/rewrite-history.php',
+ 'push' => __DIR__ . '/steps/push-repository.php',
+ 'comments' => __DIR__ . '/steps/add-comments.php',
];
-$result = [];
+$result = ['issues' => [8641 => 398]];
foreach ($steps as $number => $step) {
$result[$number] = require $step;
}
diff --git a/includes/config.php.dist b/includes/config.php.dist
index be7b499..67ef4e4 100644
--- a/includes/config.php.dist
+++ b/includes/config.php.dist
@@ -1,11 +1,17 @@
<?php
return [
+ // Temp paths
+ 'temp-path' => '/tmp/studip-svn-git',
+
// Trac
- 'trac-url' => 'https://<username>:<password>@develop.studip.de/trac/login',
- 'trac-query' => 'id=8682',
+ 'trac-url' => 'https://<username>:<password>@develop.studip.de/trac/login',
+ 'trac-query' => 'version=trunk&order=id&resolution=fixed&resolution=wontfix&resolution=duplicate&resolution=worksforme',
+ 'trac-revision' => 47000, // 43791 <- 4.0 started here
+ 'trac-users' => __DIR__ . '/trac-users.txt',
// GitLab
- 'gitlab-url' => 'http://gitlabdev01.virt.uni-oldenburg.de',
+ 'gitlab-api-url' => 'http://gitlabdev01.virt.uni-oldenburg.de',
+ 'gitlab-repo-url' => 'git@gitlabdev01.virt.uni-oldenburg.de:studip/trunk.git',
'gitlab-project' => 'studip/trunk',
'gitlab-access-token' => '<gitlab-token>',
'gitlab-admin-token' => true,
diff --git a/includes/functions.php b/includes/functions.php
new file mode 100644
index 0000000..7494944
--- /dev/null
+++ b/includes/functions.php
@@ -0,0 +1,40 @@
+<?php
+function my_exec($command) {
+ static $level = 0;
+
+ $level += 1;
+
+ if (func_num_args() === 1 && is_array($command)) {
+ return array_map(function ($args) {
+ return call_user_func_array('my_exec', $args);
+ }, $command);
+ }
+
+ $args = func_get_args();
+ $command = array_shift($args);
+ $callback = is_callable(end($args))
+ ? array_pop($args)
+ : function ($i) { return $i; };
+
+ $command = vsprintf($command, array_slice(func_get_args(), 1));
+ exec($command, $output);
+ $result = $callback(implode("\n", $output));
+
+ $level -= 1;
+
+ return $result;
+}
+
+function removeDirectory($directory)
+{
+ $it = new RecursiveDirectoryIterator($directory, RecursiveDirectoryIterator::SKIP_DOTS);
+ $files = new RecursiveIteratorIterator($it, RecursiveIteratorIterator::CHILD_FIRST);
+ foreach ($files as $file) {
+ if ($file->isDir()){
+ rmdir($file->getRealPath());
+ } else {
+ unlink($file->getRealPath());
+ }
+ }
+ rmdir($directory);
+}
diff --git a/src/GitLab.php b/src/GitLab.php
index 3069ddb..c09ae53 100644
--- a/src/GitLab.php
+++ b/src/GitLab.php
@@ -78,7 +78,7 @@ class GitLab
} catch (\Gitlab\Exception\RuntimeException $e) {
// If adding has failed because of SUDO (author does not have access to the project), create an issue without SUDO (as the Admin user whose token is configured)
if ($this->isAdmin) {
- $issue = $this->doCreateIssue($projectId, $title, $description, $createdAt, $assigneeId, $authorId, $labels, false);
+ $issue = $this->doCreateIssue($projectId, $title, $description, $createdAt, $assigneeId, $authorId, $labels, $confidential, false);
} else {
// If adding has failed for some other reason, propagate the exception back
throw $e;
@@ -206,5 +206,10 @@ class GitLab
public function getUrl() {
return $this->url;
}
+
+ public function getClient()
+ {
+ return $this->client;
+ }
}
?>
diff --git a/src/Migration.php b/src/Migration.php
index 25300b9..6d9f180 100644
--- a/src/Migration.php
+++ b/src/Migration.php
@@ -10,8 +10,8 @@ namespace Trac2GitLab;
class Migration
{
// Communicators
- private $gitLab;
- private $trac;
+ public $gitLab;
+ public $trac;
// Configuration
private $addLinkToOriginalTicket;
private $userMapping;
@@ -92,7 +92,7 @@ class Migration
* @param array $openTickets Array of Trac tickets to be migrated
* @param string $gitLabProject GitLab project in which the issues should be created
*/
- private function migrate($openTickets, $gitLabProject) {
+ public function migrate($openTickets, $gitLabProject) {
$mapping = [];
foreach($openTickets as $ticket) {
$originalTicketId = $ticket[0];
@@ -108,7 +108,7 @@ class Migration
$labels = $ticket[3]['keywords'];
$dateCreated = $ticket[3]['time']['__jsonclass__'][1];
$dateUpdated = $ticket[3]['_ts'];
- $confidential = $ticket[3]['sensitive'];
+ $confidential = (bool) @$ticket[3]['sensitive'];
$attachments = $this->trac->getAttachments($originalTicketId);
@@ -162,7 +162,7 @@ class Migration
* @return string
*/
// Adapted from: https://gitlab.dyomedea.com/vdv/trac-to-gitlab/blob/master/trac2down/Trac2Down.py
- private function translateTracToMarkdown($text) {
+ public function translateTracToMarkdown($text) {
$text = str_replace("\r\n", "\n", $text);
// Inline code block
$text = preg_replace('/{{{(.*?)}}}/', '`$1`', $text);
diff --git a/src/Trac.php b/src/Trac.php
index 45838ee..a07e4ab 100644
--- a/src/Trac.php
+++ b/src/Trac.php
@@ -48,7 +48,6 @@ class Trac
$ticketIds = $this->client->execute('ticket.query', array($query));
foreach($ticketIds as $id) {
$tickets[$id] = $this->getTicket($id);
- $tickets[$id][] = $this->getComments($id);
}
return $tickets;
}
@@ -111,5 +110,10 @@ class Trac
public function getUrl() {
return $this->url;
}
+
+ public function getClient()
+ {
+ return $this->client;
+ }
}
?>
diff --git a/steps/add-comments.php b/steps/add-comments.php
new file mode 100644
index 0000000..405e1d3
--- /dev/null
+++ b/steps/add-comments.php
@@ -0,0 +1,44 @@
+<?php
+use Trac2GitLab\Migration;
+
+$migration = new Migration(
+ $config['gitlab-api-url'],
+ $config['gitlab-access-token'],
+ $config['gitlab-admin-token'],
+ $config['trac-url'],
+ $config['create-trac-links'],
+ $userMapping = []
+);
+$trac = $migration->trac;
+$gitlab = $migration->gitLab->getClient();
+
+var_dump($trac->getClient()->execute('ticket.query', ['version=trunk&max=0']));
+die;
+
+foreach ($result['issues'] as $trac_id => $gitlab_id) {
+ $comments = $trac->getComments($trac_id);
+
+ foreach ($comments as $comment) {
+ $text = $comment['text'];
+
+
+// In [changeset:"48002"]:
+// {{{
+// #!CommitTicketReference repository="" revision="48002"
+// fixes #8641
+// }}}
+ // try {
+ // $gitlab->api('issues')->addComment($config['gitlab-project'], $gitlab_id, [
+ // 'body' => $comment['text'],
+ // 'sudo' => $comment['author'],
+ // 'created_at' => $comment['time']['__jsonclass__'][1],
+ // ]);
+ // } catch (Exception $e) {
+ $gitlab->api('issues')->addComment($config['gitlab-project'], $gitlab_id, [
+ 'body' => $text,
+ 'created_at' => $comment['time']['__jsonclass__'][1],
+ 'updated_at' => $comment['time']['__jsonclass__'][1],
+ ]);
+ // }
+ }
+}
diff --git a/steps/clone-svn-repository.php b/steps/clone-svn-repository.php
new file mode 100644
index 0000000..3c58cee
--- /dev/null
+++ b/steps/clone-svn-repository.php
@@ -0,0 +1,34 @@
+<?php
+define('LOG_REGEXP', '~^r(\d+) = ([0-9a-f]+) \(refs/remotes/git-svn\)~');
+
+// Remove old git directory
+if (file_exists($config['temp-path'])) {
+ rename($config['temp-path'], $config['temp-path'] . '-' . md5(uniqid('conversion', true)));
+}
+
+// Clone svn repository as git
+$commit_mapping = [];
+
+echo "> git svn clone...";
+my_exec(
+ 'git svn clone -r%u:HEAD --no-metadata --authors-file=%s svn://develop.studip.de/studip/trunk %s 2> /dev/null',
+ $config['trac-revision'],
+ $config['trac-users'],
+ $config['temp-path'],
+ function ($output) use (&$commit_mapping) {
+ $lines = explode("\n", $output);
+
+ foreach ($lines as $line) {
+ if (!preg_match(LOG_REGEXP, $line, $match)) {
+ continue;
+ }
+
+ $commit_mapping[$match[1]] = $match[2];
+ }
+
+ return $output;
+ }
+);
+echo "done\n";
+
+return $commit_mapping;
diff --git a/steps/convert-tickets-to-issues.php b/steps/convert-tickets-to-issues.php
index f0232b8..2307228 100644
--- a/steps/convert-tickets-to-issues.php
+++ b/steps/convert-tickets-to-issues.php
@@ -5,13 +5,29 @@ $issue_mapping = [];
// Actually migrate
$migration = new Migration(
- $config['gitlab-url'],
+ $config['gitlab-api-url'],
$config['gitlab-access-token'],
$config['gitlab-admin-token'],
$config['trac-url'],
$config['create-trac-links'],
$userMapping = []
);
+$trac = $migration->trac->getClient();
+
+$step_size = 50;
+$page = 1;
+do {
+ $query = "{$config['trac-query']}&page={$page}&max={$step_size}";
+ try {
+ $ticket_ids = $trac->execute('ticket.query', [$query]);
+ $migration->migrate($ticket_ids, $config['gitlab-project']);
+ } catch (Exception $e) {
+ $ticket_ids = [];
+ }
+
+ $page += 1;
+} while (count($ticket_ids) > 0);
+
return $migration->migrateQuery(
$config['trac-query'],
$config['gitlab-project']
diff --git a/steps/push-repository.php b/steps/push-repository.php
new file mode 100644
index 0000000..34a2b99
--- /dev/null
+++ b/steps/push-repository.php
@@ -0,0 +1,7 @@
+<?php
+echo "> Pushing repository...";
+my_exec([
+ ['git -C %s remote add origin %s', $config['temp-path'], $config['gitlab-repo-url']],
+ ['git -C %s push origin master', $config['temp-path']]
+]);
+echo "done\n";
diff --git a/steps/rewrite-history.php b/steps/rewrite-history.php
new file mode 100644
index 0000000..26723b8
--- /dev/null
+++ b/steps/rewrite-history.php
@@ -0,0 +1,53 @@
+<?php
+// Rewrite ticket numbers to issue numbers in commits
+echo "> rewrite commit messages...";
+$index = 0;
+// Skip latest entry for now
+foreach (array_reverse(array_slice($commit_mapping, 1)) as $sha1) {
+ $command = sprintf(
+ "git log -C %s --color=never --format=%%B -1 %s",
+ $config['temp-path'],
+ $sha1
+ );
+ $message = my_exec(
+ "git -C %s log --color=never --format=%%B -1 %s",
+ $config['temp-path'],
+ $sha1,
+ function ($message) use ($result) {
+ // Replace tickets/issues
+ $message = preg_replace_callback('/(?<=\s|^|,)#(\d+)\b/', function ($match) use ($result) {
+ if (isset($result['issues'][$match[1]])) {
+ return '#' . $result['issues'][$match[1]];
+ }
+ return $match[0];
+ }, $message);
+
+ // Replace changesets/commits
+ $message = preg_replace_callback('/(?<=\s|^|,)\[(\d+(?:,\d+)*)\](?=\s|$|,)/', function ($match) use ($result) {
+ $changesets = explode(',', $match[1]);
+ foreach ($changesets as $index => $changeset) {
+ if (isset($result['commits'][$changeset])) {
+ $changesets[$index] = $result['commits'][$changeset];
+ }
+ }
+
+ return '[' . implode(',', $changesets) . ']';
+ }, $message);
+
+ return $message;
+ }
+ );
+
+ my_exec([
+ ['git -C %s checkout -b amending HEAD~%u', $config['temp-path'], $index + 1],
+ ['git -C %s cherry-pick %s', $config['temp-path'], $sha1],
+ ['git -C %s commit --amend -m "%s"', $config['temp-path'], addslashes($message)],
+ ['git -C %s rebase --onto amending %s master', $config['temp-path'], $sha1],
+ ['git -C %s branch -d amending', $config['temp-path']],
+ ]);
+
+ unset($commit_mapping[$sha1]);
+
+ $index += 1;
+}
+echo "done\n";
--
GitLab