From: Mathieu Baudier Date: Mon, 2 May 2022 09:22:24 +0000 (+0200) Subject: Use IMAP for target server X-Git-Tag: v2.3.5~76 X-Git-Url: http://git.argeo.org/?a=commitdiff_plain;h=ab1c436727b53222efdc02e2fae6ea2bc63d2a4b;p=gpl%2Fargeo-slc.git Use IMAP for target server --- diff --git a/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java b/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java index 522fe7cdd..79798192a 100644 --- a/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java +++ b/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java @@ -4,22 +4,14 @@ import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; import static org.argeo.slc.mail.EmailUtils.describe; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.System.Logger; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.time.Instant; import java.util.Date; -import java.util.Enumeration; import java.util.Properties; import javax.mail.FetchProfile; @@ -29,45 +21,60 @@ import javax.mail.MessagingException; import javax.mail.Multipart; import javax.mail.Session; import javax.mail.Store; -import javax.mail.URLName; -import javax.mail.internet.InternetHeaders; import javax.mail.internet.MimeBodyPart; import javax.mail.internet.MimeMessage; import javax.mail.search.HeaderTerm; -import javax.mail.util.SharedFileInputStream; import com.sun.mail.imap.IMAPFolder; -import com.sun.mail.mbox.MboxFolder; -import com.sun.mail.mbox.MboxMessage; /** Migrates emails from one storage to the another one. */ public class EmailMigration { private final static Logger logger = System.getLogger(EmailMigration.class.getName()); - private String targetBaseDir; +// private String targetBaseDir; + private String sourceServer; private String sourceUsername; private String sourcePassword; + private String targetServer; + private String targetUsername; + private String targetPassword; + public void process() throws MessagingException, IOException { - Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox"); +// Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox"); Store sourceStore = null; try { Properties sourceProperties = System.getProperties(); sourceProperties.setProperty("mail.store.protocol", "imaps"); - Session sourceSession = Session.getDefaultInstance(sourceProperties, null); + Session sourceSession = Session.getInstance(sourceProperties, null); // session.setDebug(true); sourceStore = sourceSession.getStore("imaps"); sourceStore.connect(sourceServer, sourceUsername, sourcePassword); Folder defaultFolder = sourceStore.getDefaultFolder(); - migrateFolders(baseDir, defaultFolder); +// migrateFolders(baseDir, defaultFolder); // Always start with Inbox // Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX); // migrateFolder(baseDir, inboxFolder); + + Properties targetProperties = System.getProperties(); + targetProperties.setProperty("mail.imap.starttls.enable", "true"); + targetProperties.setProperty("mail.imap.auth", "true"); + + Session targetSession = Session.getInstance(targetProperties, null); + // session.setDebug(true); + Store targetStore = targetSession.getStore("imap"); + targetStore.connect(targetServer, targetUsername, targetPassword); + +// Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX); +// logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount()); +// logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount()); + + migrateFolders(defaultFolder, targetStore); } finally { if (sourceStore != null) sourceStore.close(); @@ -75,98 +82,161 @@ public class EmailMigration { } } - protected void migrateFolders(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { + protected void migrateFolders(Folder sourceFolder, Store targetStore) throws MessagingException, IOException { folders: for (Folder folder : sourceFolder.list()) { String folderName = folder.getName(); - if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { + String folderFullName = folder.getFullName(); + + // GMail specific + if (folderFullName.equals("[Gmail]")) { + migrateFolders(folder, targetStore); + continue folders; + } + if (folderFullName.startsWith("[Gmail]")) { // Make it configurable switch (folderName) { case "All Mail": case "Important": + case "Spam": continue folders; default: - // doe nothing + // does nothing } - migrateFolder(baseDir, folder); + folderFullName = folder.getName(); } - if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { - migrateFolders(baseDir.resolve(folder.getName()), folder); - } - } - } - - protected void migrateFolder(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { - String folderName = sourceFolder.getName(); - sourceFolder.open(Folder.READ_ONLY); + int messageCount = (folder.getType() & Folder.HOLDS_MESSAGES) != 0 ? folder.getMessageCount() : 0; + boolean hasSubFolders = (folder.getType() & Folder.HOLDS_FOLDERS) != 0 ? folder.list().length != 0 : false; + Folder targetFolder; + if (hasSubFolders) {// has sub-folders + if (messageCount == 0) { + targetFolder = targetStore.getFolder(folderFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_FOLDERS); + logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName()); + } + } else {// also has messages + Folder parentFolder = targetStore.getFolder(folderFullName); + if (!parentFolder.exists()) { + parentFolder.create(Folder.HOLDS_FOLDERS); + logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName()); + } + String miscFullName = folderFullName + "/_Misc"; + targetFolder = targetStore.getFolder(miscFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_MESSAGES); + logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); + } + } + } else {// no sub-folders + if (messageCount == 0) { // empty + logger.log(DEBUG, "Skip empty folder " + folderFullName); + continue folders; + } + targetFolder = targetStore.getFolder(folderFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_MESSAGES); + logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); + } + } - Folder targetFolder = null; - try { - int messageCount = sourceFolder.getMessageCount(); - logger.log(DEBUG, folderName + " - Message count : " + messageCount); - if (messageCount == 0) - return; -// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount()); - - boolean saveAsFiles = false; - - if (saveAsFiles) { - Message messages[] = sourceFolder.getMessages(); - - for (int i = 0; i < messages.length; ++i) { -// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":"); - Message msg = messages[i]; -// String from = "unknown"; -// if (msg.getReplyTo().length >= 1) { -// from = msg.getReplyTo()[0].toString(); -// } else if (msg.getFrom().length >= 1) { -// from = msg.getFrom()[0].toString(); -// } - String subject = msg.getSubject(); - Instant sentDate = msg.getSentDate().toInstant(); -// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")"); - String fileName = sentDate + " " + subject; - Path file = baseDir.resolve(fileName); - savePartsAsFiles(msg.getContent(), file); + if (messageCount != 0) { + + targetFolder.open(Folder.READ_WRITE); + try { + long begin = System.currentTimeMillis(); + folder.open(Folder.READ_ONLY); + migrateFolder(folder, targetFolder); + long duration = System.currentTimeMillis() - begin; + logger.log(DEBUG, folderFullName + " - Migration of " + messageCount + " messages took " + + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)"); + } finally { + folder.close(); + targetFolder.close(); } - } else { - long begin = System.currentTimeMillis(); - targetFolder = migrateFolderToMbox(baseDir, sourceFolder); - long duration = System.currentTimeMillis() - begin; - logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000) - + " s (" + (duration / messageCount) + " ms per message)"); } - } finally { - sourceFolder.close(); - if (targetFolder != null) - targetFolder.close(); + + // recursive + if (hasSubFolders) { + migrateFolders(folder, targetStore); + } } } - protected Folder migrateFolderToMbox(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { - String folderName = sourceFolder.getName(); - if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE)) - folderName = EmailUtils.INBOX;// Inbox - - Path targetDir = baseDir;// .resolve("mbox"); - Files.createDirectories(targetDir); - Path targetPath; - if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) { - Path dir = targetDir.resolve(folderName); - Files.createDirectories(dir); - targetPath = dir.resolve("_Misc"); - } else { - targetPath = targetDir.resolve(folderName); - } - if (!Files.exists(targetPath)) - Files.createFile(targetPath); - URLName targetUrlName = new URLName("mbox:" + targetPath.toString()); - Properties targetProperties = new Properties(); - // targetProperties.setProperty("mail.mime.address.strict", "false"); - Session targetSession = Session.getDefaultInstance(targetProperties); - Folder targetFolder = targetSession.getFolder(targetUrlName); - targetFolder.open(Folder.READ_WRITE); +// protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { +// folders: for (Folder folder : sourceFolder.list()) { +// String folderName = folder.getName(); +// +// if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { +// // Make it configurable +// switch (folderName) { +// case "All Mail": +// case "Important": +// continue folders; +// default: +// // doe nothing +// } +// migrateFolderToFs(baseDir, folder); +// } +// if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { +// migrateFoldersToFs(baseDir.resolve(folder.getName()), folder); +// } +// } +// } + +// protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { +// +// String folderName = sourceFolder.getName(); +// sourceFolder.open(Folder.READ_ONLY); +// +// Folder targetFolder = null; +// try { +// int messageCount = sourceFolder.getMessageCount(); +// logger.log(DEBUG, folderName + " - Message count : " + messageCount); +// if (messageCount == 0) +// return; +//// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount()); +// +// boolean saveAsFiles = false; +// +// if (saveAsFiles) { +// Message messages[] = sourceFolder.getMessages(); +// +// for (int i = 0; i < messages.length; ++i) { +//// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":"); +// Message msg = messages[i]; +//// String from = "unknown"; +//// if (msg.getReplyTo().length >= 1) { +//// from = msg.getReplyTo()[0].toString(); +//// } else if (msg.getFrom().length >= 1) { +//// from = msg.getFrom()[0].toString(); +//// } +// String subject = msg.getSubject(); +// Instant sentDate = msg.getSentDate().toInstant(); +//// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")"); +// String fileName = sentDate + " " + subject; +// Path file = baseDir.resolve(fileName); +// savePartsAsFiles(msg.getContent(), file); +// } +// } +// else { +// long begin = System.currentTimeMillis(); +// targetFolder = openMboxTargetFolder(sourceFolder, baseDir); +// migrateFolder(sourceFolder, targetFolder); +// long duration = System.currentTimeMillis() - begin; +// logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000) +// + " s (" + (duration / messageCount) + " ms per message)"); +// } +// } finally { +// sourceFolder.close(); +// if (targetFolder != null) +// targetFolder.close(); +// } +// } + + protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException { + String folderName = targetFolder.getName(); int lastSourceNumber; int currentTargetMessageCount = targetFolder.getMessageCount(); @@ -193,13 +263,6 @@ public class EmailMigration { logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber); int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber; -// for (int i = startNumber; i < messageCount; i++) { -// long begin = System.currentTimeMillis(); -// Message message = sourceFolder.getMessage(i); -// targetFolder.appendMessages(new Message[] { message }); -// long duration = System.currentTimeMillis() - begin; -// logger.log(DEBUG, "Message " + i + " migrated in " + duration + " ms"); -// } FetchProfile fetchProfile = new FetchProfile(); fetchProfile.add(FetchProfile.Item.FLAGS); @@ -229,69 +292,8 @@ public class EmailMigration { // targetFolder.appendMessages(sourceMessages); // sourceFolder.copyMessages(sourceMessages,targetFolder); - Message[] targetMessages = new Message[sourceMessages.length]; - for (int j = 0; j < sourceMessages.length; j++) { - MimeMessage sourceMm = (MimeMessage) sourceMessages[j]; - InternetHeaders ih = new InternetHeaders(); - for (Enumeration e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) { - ih.addHeaderLine(e.nextElement()); - } -// Flags flags = sourceMm.getFlags(); -// StringBuilder status = new StringBuilder(); -// if (flags.contains(Flags.Flag.SEEN)) -// status.append('R'); -// if (!flags.contains(Flags.Flag.RECENT)) -// status.append('O'); -// if (status.length() > 0 && ih.getHeader("X-Status") == null) -// ih.setHeader("X-Status", status.toString()); - - Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt"); - Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt"); - // logger.log(DEBUG, "tmpFileSource " + tmpFileSource + ", tmpFileTarget " + - // tmpFileTarget); - Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING); - - // we use ISO_8859_1 because it is more robust than US_ASCII with regard to - // missing characters - try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1); - BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) { - int lineNumber = 0; - String line = null; - try { - while ((line = reader.readLine()) != null) { - lineNumber++; - if (line.startsWith("From ")) { - writer.write(">" + line); - logger.log(DEBUG, "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) - + ": " + line); - } else { - writer.write(line); - } - writer.newLine(); - } - } catch (IOException e) { - logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource); - throw e; - } - } - - MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih, - new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(), - EmailUtils.getUnixFrom(sourceMm), true); - targetMessages[j] = mboxMessage; - - // clean up - Files.delete(tmpFileSource); - Files.delete(tmpFileTarget); - } - targetFolder.appendMessages(targetMessages); -// Message[] targetMessages = targetFolder.getMessages(start, end); -// for (int j = 0; j < sourceMessages.length; j++) { -// EmailUtils.setHeadersFromFlags((MimeMessage) targetMessages[j], sourceMessages[j].getFlags()); -//// Flags flags = sourceMessages[j].getFlags(); -//// targetMessages[j].setFlags(flags, true); -// targetMessages[j].saveChanges(); -// } + copyMessages(sourceMessages, targetFolder); +// copyMessagesToMbox(sourceMessages, targetFolder); String describeLast = describe(sourceMessages[sourceMessages.length - 1]); @@ -313,6 +315,87 @@ public class EmailMigration { return targetFolder; } +// protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException { +// String folderName = sourceFolder.getName(); +// if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE)) +// folderName = EmailUtils.INBOX;// Inbox +// +// Path targetDir = baseDir;// .resolve("mbox"); +// Files.createDirectories(targetDir); +// Path targetPath; +// if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) { +// Path dir = targetDir.resolve(folderName); +// Files.createDirectories(dir); +// targetPath = dir.resolve("_Misc"); +// } else { +// targetPath = targetDir.resolve(folderName); +// } +// if (!Files.exists(targetPath)) +// Files.createFile(targetPath); +// URLName targetUrlName = new URLName("mbox:" + targetPath.toString()); +// Properties targetProperties = new Properties(); +// // targetProperties.setProperty("mail.mime.address.strict", "false"); +// Session targetSession = Session.getDefaultInstance(targetProperties); +// Folder targetFolder = targetSession.getFolder(targetUrlName); +// targetFolder.open(Folder.READ_WRITE); +// +// return targetFolder; +// } + + protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException { + targetFolder.appendMessages(sourceMessages); + } + +// protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder) +// throws MessagingException, IOException { +// Message[] targetMessages = new Message[sourceMessages.length]; +// for (int j = 0; j < sourceMessages.length; j++) { +// MimeMessage sourceMm = (MimeMessage) sourceMessages[j]; +// InternetHeaders ih = new InternetHeaders(); +// for (Enumeration e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) { +// ih.addHeaderLine(e.nextElement()); +// } +// Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt"); +// Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt"); +// Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING); +// +// // we use ISO_8859_1 because it is more robust than US_ASCII with regard to +// // missing characters +// try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1); +// BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) { +// int lineNumber = 0; +// String line = null; +// try { +// while ((line = reader.readLine()) != null) { +// lineNumber++; +// if (line.startsWith("From ")) { +// writer.write(">" + line); +// logger.log(DEBUG, +// "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line); +// } else { +// writer.write(line); +// } +// writer.newLine(); +// } +// } catch (IOException e) { +// logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource); +// throw e; +// } +// } +// +// MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih, +// new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(), +// EmailUtils.getUnixFrom(sourceMm), true); +// targetMessages[j] = mboxMessage; +// +// // clean up +// Files.delete(tmpFileSource); +// Files.delete(tmpFileTarget); +// } +// targetFolder.appendMessages(targetMessages); +// +// } + /** Save body parts and attachments as plain files. */ protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException { OutputStream out = null; @@ -360,10 +443,6 @@ public class EmailMigration { } } - public void setTargetBaseDir(String targetBaseDir) { - this.targetBaseDir = targetBaseDir; - } - public void setSourceServer(String sourceServer) { this.sourceServer = sourceServer; } @@ -376,20 +455,36 @@ public class EmailMigration { this.sourcePassword = sourcePassword; } + public void setTargetServer(String targetServer) { + this.targetServer = targetServer; + } + + public void setTargetUsername(String targetUsername) { + this.targetUsername = targetUsername; + } + + public void setTargetPassword(String targetPassword) { + this.targetPassword = targetPassword; + } + public static void main(String args[]) throws Exception { - if (args.length < 4) + if (args.length < 6) throw new IllegalArgumentException( - "usage: "); - String targetBaseDir = args[0]; - String sourceServer = args[1]; - String sourceUsername = args[2]; - String sourcePassword = args[3]; + "usage: "); + String sourceServer = args[0]; + String sourceUsername = args[1]; + String sourcePassword = args[2]; + String targetServer = args[3]; + String targetUsername = args[4]; + String targetPassword = args[5]; EmailMigration emailMigration = new EmailMigration(); - emailMigration.setTargetBaseDir(targetBaseDir); emailMigration.setSourceServer(sourceServer); emailMigration.setSourceUsername(sourceUsername); emailMigration.setSourcePassword(sourcePassword); + emailMigration.setTargetServer(targetServer); + emailMigration.setTargetUsername(targetUsername); + emailMigration.setTargetPassword(targetPassword); emailMigration.process(); }