X-Git-Url: https://git.argeo.org/?p=gpl%2Fargeo-suite.git;a=blobdiff_plain;f=org.argeo.app.core%2Fsrc%2Forg%2Fargeo%2Fapp%2Fmail%2FEmailMigration.java;fp=org.argeo.app.core%2Fsrc%2Forg%2Fargeo%2Fapp%2Fmail%2FEmailMigration.java;h=0000000000000000000000000000000000000000;hp=8c899c2316ccb4dbb18433dc8501ec5b63bd5e35;hb=d7da00bfd0ae230b97abccb2cc9d091e34c11632;hpb=26edd36fbbd565d7459a134a20a9c60d489dbb35 diff --git a/org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java b/org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java deleted file mode 100644 index 8c899c2..0000000 --- a/org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java +++ /dev/null @@ -1,524 +0,0 @@ -package org.argeo.app.mail; - -import static java.lang.System.Logger.Level.DEBUG; -import static java.lang.System.Logger.Level.ERROR; -import static org.argeo.app.mail.EmailUtils.describe; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.System.Logger; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.time.Instant; -import java.util.Date; -import java.util.Enumeration; -import java.util.Properties; - -import javax.mail.FetchProfile; -import javax.mail.Folder; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.mail.Multipart; -import javax.mail.Session; -import javax.mail.Store; -import javax.mail.URLName; -import javax.mail.internet.InternetHeaders; -import javax.mail.internet.MimeBodyPart; -import javax.mail.internet.MimeMessage; -import javax.mail.search.HeaderTerm; -import javax.mail.util.SharedFileInputStream; - -import com.sun.mail.imap.IMAPFolder; -import com.sun.mail.mbox.MboxFolder; -import com.sun.mail.mbox.MboxMessage; - -/** Migrates emails from one storage to the another one. */ -public class EmailMigration { - private final static Logger logger = System.getLogger(EmailMigration.class.getName()); - -// private String targetBaseDir; - - private String sourceServer; - private String sourceUsername; - private String sourcePassword; - - private String targetServer; - private String targetUsername; - private String targetPassword; - - private boolean targetSupportDualTypeFolders = true; - - public void process() throws MessagingException, IOException { -// Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox"); - - Store sourceStore = null; - try { - Properties sourceProperties = System.getProperties(); - sourceProperties.setProperty("mail.store.protocol", "imaps"); - - Session sourceSession = Session.getInstance(sourceProperties, null); - // session.setDebug(true); - sourceStore = sourceSession.getStore("imaps"); - sourceStore.connect(sourceServer, sourceUsername, sourcePassword); - - Folder defaultFolder = sourceStore.getDefaultFolder(); -// migrateFolders(baseDir, defaultFolder); - - // Always start with Inbox -// Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX); -// migrateFolder(baseDir, inboxFolder); - - Properties targetProperties = System.getProperties(); - targetProperties.setProperty("mail.imap.starttls.enable", "true"); - targetProperties.setProperty("mail.imap.auth", "true"); - - Session targetSession = Session.getInstance(targetProperties, null); - // session.setDebug(true); - Store targetStore = targetSession.getStore("imap"); - targetStore.connect(targetServer, targetUsername, targetPassword); - -// Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX); -// logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount()); -// logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount()); - - migrateFolders(defaultFolder, targetStore); - } finally { - if (sourceStore != null) - sourceStore.close(); - - } - } - - protected void migrateFolders(Folder sourceParentFolder, Store targetStore) throws MessagingException, IOException { - folders: for (Folder sourceFolder : sourceParentFolder.list()) { - String sourceFolderName = sourceFolder.getName(); - - String sourceFolderFullName = sourceFolder.getFullName(); - char sourceFolderSeparator = sourceParentFolder.getSeparator(); - char targetFolderSeparator = targetStore.getDefaultFolder().getSeparator(); - String targetFolderFullName = sourceFolderFullName.replace(sourceFolderSeparator, targetFolderSeparator); - - // GMail specific - if (sourceFolderFullName.equals("[Gmail]")) { - migrateFolders(sourceFolder, targetStore); - continue folders; - } - if (sourceFolderFullName.startsWith("[Gmail]")) { - String subFolderName = null; - // Make it configurable - switch (sourceFolderName) { - case "All Mail": - case "Important": - case "Spam": - continue folders; - case "Sent Mail": - subFolderName = "Sent"; - default: - // does nothing - } - targetFolderFullName = subFolderName == null ? sourceFolder.getName() : subFolderName; - } - - // nature of the source folder - int messageCount = (sourceFolder.getType() & Folder.HOLDS_MESSAGES) != 0 ? sourceFolder.getMessageCount() - : 0; - boolean hasSubFolders = (sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0 - ? sourceFolder.list().length != 0 - : false; - - Folder targetFolder; - if (targetSupportDualTypeFolders) { - targetFolder = targetStore.getFolder(targetFolderFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_FOLDERS | Folder.HOLDS_MESSAGES); - logger.log(DEBUG, "Created HOLDS_FOLDERS | HOLDS_MESSAGES folder " + targetFolder.getFullName()); - } - - } else { - if (hasSubFolders) {// has sub-folders - if (messageCount == 0) { - targetFolder = targetStore.getFolder(targetFolderFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_FOLDERS); - logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName()); - } - } else {// also has messages - Folder parentFolder = targetStore.getFolder(targetFolderFullName); - if (!parentFolder.exists()) { - parentFolder.create(Folder.HOLDS_FOLDERS); - logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName()); - } - String miscFullName = targetFolderFullName + targetFolderSeparator + "_Misc"; - targetFolder = targetStore.getFolder(miscFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_MESSAGES); - logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); - } - } - } else {// no sub-folders - if (messageCount == 0) { // empty - logger.log(DEBUG, "Skip empty folder " + targetFolderFullName); - continue folders; - } - targetFolder = targetStore.getFolder(targetFolderFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_MESSAGES); - logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); - } - } - } - - if (messageCount != 0) { - - targetFolder.open(Folder.READ_WRITE); - try { - long begin = System.currentTimeMillis(); - sourceFolder.open(Folder.READ_ONLY); - migrateFolder(sourceFolder, targetFolder); - long duration = System.currentTimeMillis() - begin; - logger.log(DEBUG, targetFolderFullName + " - Migration of " + messageCount + " messages took " - + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)"); - } finally { - sourceFolder.close(); - targetFolder.close(); - } - } - - // recursive - if (hasSubFolders) { - migrateFolders(sourceFolder, targetStore); - } - } - } - - protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { - folders: for (Folder folder : sourceFolder.list()) { - String folderName = folder.getName(); - - if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { - // Make it configurable - switch (folderName) { - case "All Mail": - case "Important": - continue folders; - default: - // doe nothing - } - migrateFolderToFs(baseDir, folder); - } - if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { - migrateFoldersToFs(baseDir.resolve(folder.getName()), folder); - } - } - } - - protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { - - String folderName = sourceFolder.getName(); - sourceFolder.open(Folder.READ_ONLY); - - Folder targetFolder = null; - try { - int messageCount = sourceFolder.getMessageCount(); - logger.log(DEBUG, folderName + " - Message count : " + messageCount); - if (messageCount == 0) - return; -// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount()); - - boolean saveAsFiles = false; - - if (saveAsFiles) { - Message messages[] = sourceFolder.getMessages(); - - for (int i = 0; i < messages.length; ++i) { -// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":"); - Message msg = messages[i]; -// String from = "unknown"; -// if (msg.getReplyTo().length >= 1) { -// from = msg.getReplyTo()[0].toString(); -// } else if (msg.getFrom().length >= 1) { -// from = msg.getFrom()[0].toString(); -// } - String subject = msg.getSubject(); - Instant sentDate = msg.getSentDate().toInstant(); -// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")"); - String fileName = sentDate + " " + subject; - Path file = baseDir.resolve(fileName); - savePartsAsFiles(msg.getContent(), file); - } - } else { - long begin = System.currentTimeMillis(); - targetFolder = openMboxTargetFolder(sourceFolder, baseDir); - migrateFolder(sourceFolder, targetFolder); - long duration = System.currentTimeMillis() - begin; - logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000) - + " s (" + (duration / messageCount) + " ms per message)"); - } - } finally { - sourceFolder.close(); - if (targetFolder != null) - targetFolder.close(); - } - } - - protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException { - String folderName = targetFolder.getName(); - - int lastSourceNumber; - int currentTargetMessageCount = targetFolder.getMessageCount(); - if (currentTargetMessageCount != 0) { - MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount); - logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage)); - Date lastTargetSent = lastTargetMessage.getReceivedDate(); - Message[] lastSourceMessage = sourceFolder - .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID())); - if (lastSourceMessage.length == 0) - throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID()); - if (lastSourceMessage.length != 1) { - for (Message msg : lastSourceMessage) { - logger.log(ERROR, "Message " + describe(msg)); - - } - throw new IllegalStateException( - lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant()); - } - lastSourceNumber = lastSourceMessage[0].getMessageNumber(); - } else { - lastSourceNumber = 0; - } - logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber); - - int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber; - - FetchProfile fetchProfile = new FetchProfile(); - fetchProfile.add(FetchProfile.Item.FLAGS); - fetchProfile.add(FetchProfile.Item.ENVELOPE); - fetchProfile.add(FetchProfile.Item.CONTENT_INFO); - fetchProfile.add(FetchProfile.Item.SIZE); - if (sourceFolder instanceof IMAPFolder) { - // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder; - fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS); - fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE); - } - - int batchSize = 100; - int batchCount = countToRetrieve / batchSize; - if (countToRetrieve % batchSize != 0) - batchCount = batchCount + 1; - // int batchCount = 2; // for testing - for (int i = 0; i < batchCount; i++) { - long begin = System.currentTimeMillis(); - - int start = lastSourceNumber + i * batchSize + 1; - int end = lastSourceNumber + (i + 1) * batchSize; - if (end >= (lastSourceNumber + countToRetrieve + 1)) - end = lastSourceNumber + countToRetrieve; - Message[] sourceMessages = sourceFolder.getMessages(start, end); - sourceFolder.fetch(sourceMessages, fetchProfile); - // targetFolder.appendMessages(sourceMessages); - // sourceFolder.copyMessages(sourceMessages,targetFolder); - - copyMessages(sourceMessages, targetFolder); -// copyMessagesToMbox(sourceMessages, targetFolder); - - String describeLast = describe(sourceMessages[sourceMessages.length - 1]); - -// if (i % 10 == 9) { - // free memory from fetched messages - sourceFolder.close(); - targetFolder.close(); - - sourceFolder.open(Folder.READ_ONLY); - targetFolder.open(Folder.READ_WRITE); -// logger.log(DEBUG, "Open/close folder in order to free memory"); -// } - - long duration = System.currentTimeMillis() - begin; - logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, " - + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast); - } - - return targetFolder; - } - - protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException { - String folderName = sourceFolder.getName(); - if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE)) - folderName = EmailUtils.INBOX;// Inbox - - Path targetDir = baseDir;// .resolve("mbox"); - Files.createDirectories(targetDir); - Path targetPath; - if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) { - Path dir = targetDir.resolve(folderName); - Files.createDirectories(dir); - targetPath = dir.resolve("_Misc"); - } else { - targetPath = targetDir.resolve(folderName); - } - if (!Files.exists(targetPath)) - Files.createFile(targetPath); - URLName targetUrlName = new URLName("mbox:" + targetPath.toString()); - Properties targetProperties = new Properties(); - // targetProperties.setProperty("mail.mime.address.strict", "false"); - Session targetSession = Session.getDefaultInstance(targetProperties); - Folder targetFolder = targetSession.getFolder(targetUrlName); - targetFolder.open(Folder.READ_WRITE); - - return targetFolder; - } - - protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException { - targetFolder.appendMessages(sourceMessages); - } - - protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder) - throws MessagingException, IOException { - Message[] targetMessages = new Message[sourceMessages.length]; - for (int j = 0; j < sourceMessages.length; j++) { - MimeMessage sourceMm = (MimeMessage) sourceMessages[j]; - InternetHeaders ih = new InternetHeaders(); - for (Enumeration e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) { - ih.addHeaderLine(e.nextElement()); - } - Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt"); - Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt"); - Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING); - - // we use ISO_8859_1 because it is more robust than US_ASCII with regard to - // missing characters - try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1); - BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) { - int lineNumber = 0; - String line = null; - try { - while ((line = reader.readLine()) != null) { - lineNumber++; - if (line.startsWith("From ")) { - writer.write(">" + line); - logger.log(DEBUG, - "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line); - } else { - writer.write(line); - } - writer.newLine(); - } - } catch (IOException e) { - logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource); - throw e; - } - } - - MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih, - new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(), - EmailUtils.getUnixFrom(sourceMm), true); - targetMessages[j] = mboxMessage; - - // clean up - Files.delete(tmpFileSource); - Files.delete(tmpFileTarget); - } - targetFolder.appendMessages(targetMessages); - - } - - /** Save body parts and attachments as plain files. */ - protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException { - OutputStream out = null; - InputStream in = null; - try { - if (content instanceof Multipart) { - Multipart multi = ((Multipart) content); - int parts = multi.getCount(); - for (int j = 0; j < parts; ++j) { - MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j); - if (part.getContent() instanceof Multipart) { - // part-within-a-part, do some recursion... - savePartsAsFiles(part.getContent(), fileBase); - } else { - String extension = ""; - if (part.isMimeType("text/html")) { - extension = "html"; - } else { - if (part.isMimeType("text/plain")) { - extension = "txt"; - } else { - // Try to get the name of the attachment - extension = part.getDataHandler().getName(); - } - } - String filename = fileBase + "." + extension; - System.out.println("... " + filename); - out = new FileOutputStream(new File(filename)); - in = part.getInputStream(); - int k; - while ((k = in.read()) != -1) { - out.write(k); - } - } - } - } - } finally { - if (in != null) { - in.close(); - } - if (out != null) { - out.flush(); - out.close(); - } - } - } - - public void setSourceServer(String sourceServer) { - this.sourceServer = sourceServer; - } - - public void setSourceUsername(String sourceUsername) { - this.sourceUsername = sourceUsername; - } - - public void setSourcePassword(String sourcePassword) { - this.sourcePassword = sourcePassword; - } - - public void setTargetServer(String targetServer) { - this.targetServer = targetServer; - } - - public void setTargetUsername(String targetUsername) { - this.targetUsername = targetUsername; - } - - public void setTargetPassword(String targetPassword) { - this.targetPassword = targetPassword; - } - - public static void main(String args[]) throws Exception { - if (args.length < 6) - throw new IllegalArgumentException( - "usage: "); - String sourceServer = args[0]; - String sourceUsername = args[1]; - String sourcePassword = args[2]; - String targetServer = args[3]; - String targetUsername = args[4]; - String targetPassword = args[5]; - - EmailMigration emailMigration = new EmailMigration(); - emailMigration.setSourceServer(sourceServer); - emailMigration.setSourceUsername(sourceUsername); - emailMigration.setSourcePassword(sourcePassword); - emailMigration.setTargetServer(targetServer); - emailMigration.setTargetUsername(targetUsername); - emailMigration.setTargetPassword(targetPassword); - - emailMigration.process(); - } -}