From: Mathieu Baudier Date: Wed, 4 May 2022 09:39:15 +0000 (+0200) Subject: Move mail to SLC runtime X-Git-Tag: v2.3.5~74 X-Git-Url: http://git.argeo.org/?a=commitdiff_plain;ds=inline;h=984364375edbe5919b5bc30b3e39783f25f3640e;p=gpl%2Fargeo-slc.git Move mail to SLC runtime --- diff --git a/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java b/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java deleted file mode 100644 index 79798192a..000000000 --- a/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java +++ /dev/null @@ -1,491 +0,0 @@ -package org.argeo.slc.mail; - -import static java.lang.System.Logger.Level.DEBUG; -import static java.lang.System.Logger.Level.ERROR; -import static org.argeo.slc.mail.EmailUtils.describe; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.System.Logger; -import java.nio.file.Path; -import java.util.Date; -import java.util.Properties; - -import javax.mail.FetchProfile; -import javax.mail.Folder; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.mail.Multipart; -import javax.mail.Session; -import javax.mail.Store; -import javax.mail.internet.MimeBodyPart; -import javax.mail.internet.MimeMessage; -import javax.mail.search.HeaderTerm; - -import com.sun.mail.imap.IMAPFolder; - -/** Migrates emails from one storage to the another one. */ -public class EmailMigration { - private final static Logger logger = System.getLogger(EmailMigration.class.getName()); - -// private String targetBaseDir; - - private String sourceServer; - private String sourceUsername; - private String sourcePassword; - - private String targetServer; - private String targetUsername; - private String targetPassword; - - public void process() throws MessagingException, IOException { -// Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox"); - - Store sourceStore = null; - try { - Properties sourceProperties = System.getProperties(); - sourceProperties.setProperty("mail.store.protocol", "imaps"); - - Session sourceSession = Session.getInstance(sourceProperties, null); - // session.setDebug(true); - sourceStore = sourceSession.getStore("imaps"); - sourceStore.connect(sourceServer, sourceUsername, sourcePassword); - - Folder defaultFolder = sourceStore.getDefaultFolder(); -// migrateFolders(baseDir, defaultFolder); - - // Always start with Inbox -// Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX); -// migrateFolder(baseDir, inboxFolder); - - Properties targetProperties = System.getProperties(); - targetProperties.setProperty("mail.imap.starttls.enable", "true"); - targetProperties.setProperty("mail.imap.auth", "true"); - - Session targetSession = Session.getInstance(targetProperties, null); - // session.setDebug(true); - Store targetStore = targetSession.getStore("imap"); - targetStore.connect(targetServer, targetUsername, targetPassword); - -// Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX); -// logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount()); -// logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount()); - - migrateFolders(defaultFolder, targetStore); - } finally { - if (sourceStore != null) - sourceStore.close(); - - } - } - - protected void migrateFolders(Folder sourceFolder, Store targetStore) throws MessagingException, IOException { - folders: for (Folder folder : sourceFolder.list()) { - String folderName = folder.getName(); - - String folderFullName = folder.getFullName(); - - // GMail specific - if (folderFullName.equals("[Gmail]")) { - migrateFolders(folder, targetStore); - continue folders; - } - if (folderFullName.startsWith("[Gmail]")) { - // Make it configurable - switch (folderName) { - case "All Mail": - case "Important": - case "Spam": - continue folders; - default: - // does nothing - } - folderFullName = folder.getName(); - } - - int messageCount = (folder.getType() & Folder.HOLDS_MESSAGES) != 0 ? folder.getMessageCount() : 0; - boolean hasSubFolders = (folder.getType() & Folder.HOLDS_FOLDERS) != 0 ? folder.list().length != 0 : false; - Folder targetFolder; - if (hasSubFolders) {// has sub-folders - if (messageCount == 0) { - targetFolder = targetStore.getFolder(folderFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_FOLDERS); - logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName()); - } - } else {// also has messages - Folder parentFolder = targetStore.getFolder(folderFullName); - if (!parentFolder.exists()) { - parentFolder.create(Folder.HOLDS_FOLDERS); - logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName()); - } - String miscFullName = folderFullName + "/_Misc"; - targetFolder = targetStore.getFolder(miscFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_MESSAGES); - logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); - } - } - } else {// no sub-folders - if (messageCount == 0) { // empty - logger.log(DEBUG, "Skip empty folder " + folderFullName); - continue folders; - } - targetFolder = targetStore.getFolder(folderFullName); - if (!targetFolder.exists()) { - targetFolder.create(Folder.HOLDS_MESSAGES); - logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); - } - } - - if (messageCount != 0) { - - targetFolder.open(Folder.READ_WRITE); - try { - long begin = System.currentTimeMillis(); - folder.open(Folder.READ_ONLY); - migrateFolder(folder, targetFolder); - long duration = System.currentTimeMillis() - begin; - logger.log(DEBUG, folderFullName + " - Migration of " + messageCount + " messages took " - + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)"); - } finally { - folder.close(); - targetFolder.close(); - } - } - - // recursive - if (hasSubFolders) { - migrateFolders(folder, targetStore); - } - } - } - -// protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { -// folders: for (Folder folder : sourceFolder.list()) { -// String folderName = folder.getName(); -// -// if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { -// // Make it configurable -// switch (folderName) { -// case "All Mail": -// case "Important": -// continue folders; -// default: -// // doe nothing -// } -// migrateFolderToFs(baseDir, folder); -// } -// if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { -// migrateFoldersToFs(baseDir.resolve(folder.getName()), folder); -// } -// } -// } - -// protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { -// -// String folderName = sourceFolder.getName(); -// sourceFolder.open(Folder.READ_ONLY); -// -// Folder targetFolder = null; -// try { -// int messageCount = sourceFolder.getMessageCount(); -// logger.log(DEBUG, folderName + " - Message count : " + messageCount); -// if (messageCount == 0) -// return; -//// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount()); -// -// boolean saveAsFiles = false; -// -// if (saveAsFiles) { -// Message messages[] = sourceFolder.getMessages(); -// -// for (int i = 0; i < messages.length; ++i) { -//// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":"); -// Message msg = messages[i]; -//// String from = "unknown"; -//// if (msg.getReplyTo().length >= 1) { -//// from = msg.getReplyTo()[0].toString(); -//// } else if (msg.getFrom().length >= 1) { -//// from = msg.getFrom()[0].toString(); -//// } -// String subject = msg.getSubject(); -// Instant sentDate = msg.getSentDate().toInstant(); -//// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")"); -// String fileName = sentDate + " " + subject; -// Path file = baseDir.resolve(fileName); -// savePartsAsFiles(msg.getContent(), file); -// } -// } -// else { -// long begin = System.currentTimeMillis(); -// targetFolder = openMboxTargetFolder(sourceFolder, baseDir); -// migrateFolder(sourceFolder, targetFolder); -// long duration = System.currentTimeMillis() - begin; -// logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000) -// + " s (" + (duration / messageCount) + " ms per message)"); -// } -// } finally { -// sourceFolder.close(); -// if (targetFolder != null) -// targetFolder.close(); -// } -// } - - protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException { - String folderName = targetFolder.getName(); - - int lastSourceNumber; - int currentTargetMessageCount = targetFolder.getMessageCount(); - if (currentTargetMessageCount != 0) { - MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount); - logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage)); - Date lastTargetSent = lastTargetMessage.getReceivedDate(); - Message[] lastSourceMessage = sourceFolder - .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID())); - if (lastSourceMessage.length == 0) - throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID()); - if (lastSourceMessage.length != 1) { - for (Message msg : lastSourceMessage) { - logger.log(ERROR, "Message " + describe(msg)); - - } - throw new IllegalStateException( - lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant()); - } - lastSourceNumber = lastSourceMessage[0].getMessageNumber(); - } else { - lastSourceNumber = 0; - } - logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber); - - int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber; - - FetchProfile fetchProfile = new FetchProfile(); - fetchProfile.add(FetchProfile.Item.FLAGS); - fetchProfile.add(FetchProfile.Item.ENVELOPE); - fetchProfile.add(FetchProfile.Item.CONTENT_INFO); - fetchProfile.add(FetchProfile.Item.SIZE); - if (sourceFolder instanceof IMAPFolder) { - // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder; - fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS); - fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE); - } - - int batchSize = 100; - int batchCount = countToRetrieve / batchSize; - if (countToRetrieve % batchSize != 0) - batchCount = batchCount + 1; - // int batchCount = 2; // for testing - for (int i = 0; i < batchCount; i++) { - long begin = System.currentTimeMillis(); - - int start = lastSourceNumber + i * batchSize + 1; - int end = lastSourceNumber + (i + 1) * batchSize; - if (end >= (lastSourceNumber + countToRetrieve + 1)) - end = lastSourceNumber + countToRetrieve; - Message[] sourceMessages = sourceFolder.getMessages(start, end); - sourceFolder.fetch(sourceMessages, fetchProfile); - // targetFolder.appendMessages(sourceMessages); - // sourceFolder.copyMessages(sourceMessages,targetFolder); - - copyMessages(sourceMessages, targetFolder); -// copyMessagesToMbox(sourceMessages, targetFolder); - - String describeLast = describe(sourceMessages[sourceMessages.length - 1]); - -// if (i % 10 == 9) { - // free memory from fetched messages - sourceFolder.close(); - targetFolder.close(); - - sourceFolder.open(Folder.READ_ONLY); - targetFolder.open(Folder.READ_WRITE); -// logger.log(DEBUG, "Open/close folder in order to free memory"); -// } - - long duration = System.currentTimeMillis() - begin; - logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, " - + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast); - } - - return targetFolder; - } - -// protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException { -// String folderName = sourceFolder.getName(); -// if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE)) -// folderName = EmailUtils.INBOX;// Inbox -// -// Path targetDir = baseDir;// .resolve("mbox"); -// Files.createDirectories(targetDir); -// Path targetPath; -// if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) { -// Path dir = targetDir.resolve(folderName); -// Files.createDirectories(dir); -// targetPath = dir.resolve("_Misc"); -// } else { -// targetPath = targetDir.resolve(folderName); -// } -// if (!Files.exists(targetPath)) -// Files.createFile(targetPath); -// URLName targetUrlName = new URLName("mbox:" + targetPath.toString()); -// Properties targetProperties = new Properties(); -// // targetProperties.setProperty("mail.mime.address.strict", "false"); -// Session targetSession = Session.getDefaultInstance(targetProperties); -// Folder targetFolder = targetSession.getFolder(targetUrlName); -// targetFolder.open(Folder.READ_WRITE); -// -// return targetFolder; -// } - - protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException { - targetFolder.appendMessages(sourceMessages); - } - -// protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder) -// throws MessagingException, IOException { -// Message[] targetMessages = new Message[sourceMessages.length]; -// for (int j = 0; j < sourceMessages.length; j++) { -// MimeMessage sourceMm = (MimeMessage) sourceMessages[j]; -// InternetHeaders ih = new InternetHeaders(); -// for (Enumeration e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) { -// ih.addHeaderLine(e.nextElement()); -// } -// Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt"); -// Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt"); -// Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING); -// -// // we use ISO_8859_1 because it is more robust than US_ASCII with regard to -// // missing characters -// try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1); -// BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) { -// int lineNumber = 0; -// String line = null; -// try { -// while ((line = reader.readLine()) != null) { -// lineNumber++; -// if (line.startsWith("From ")) { -// writer.write(">" + line); -// logger.log(DEBUG, -// "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line); -// } else { -// writer.write(line); -// } -// writer.newLine(); -// } -// } catch (IOException e) { -// logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource); -// throw e; -// } -// } -// -// MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih, -// new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(), -// EmailUtils.getUnixFrom(sourceMm), true); -// targetMessages[j] = mboxMessage; -// -// // clean up -// Files.delete(tmpFileSource); -// Files.delete(tmpFileTarget); -// } -// targetFolder.appendMessages(targetMessages); -// -// } - - /** Save body parts and attachments as plain files. */ - protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException { - OutputStream out = null; - InputStream in = null; - try { - if (content instanceof Multipart) { - Multipart multi = ((Multipart) content); - int parts = multi.getCount(); - for (int j = 0; j < parts; ++j) { - MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j); - if (part.getContent() instanceof Multipart) { - // part-within-a-part, do some recursion... - savePartsAsFiles(part.getContent(), fileBase); - } else { - String extension = ""; - if (part.isMimeType("text/html")) { - extension = "html"; - } else { - if (part.isMimeType("text/plain")) { - extension = "txt"; - } else { - // Try to get the name of the attachment - extension = part.getDataHandler().getName(); - } - } - String filename = fileBase + "." + extension; - System.out.println("... " + filename); - out = new FileOutputStream(new File(filename)); - in = part.getInputStream(); - int k; - while ((k = in.read()) != -1) { - out.write(k); - } - } - } - } - } finally { - if (in != null) { - in.close(); - } - if (out != null) { - out.flush(); - out.close(); - } - } - } - - public void setSourceServer(String sourceServer) { - this.sourceServer = sourceServer; - } - - public void setSourceUsername(String sourceUsername) { - this.sourceUsername = sourceUsername; - } - - public void setSourcePassword(String sourcePassword) { - this.sourcePassword = sourcePassword; - } - - public void setTargetServer(String targetServer) { - this.targetServer = targetServer; - } - - public void setTargetUsername(String targetUsername) { - this.targetUsername = targetUsername; - } - - public void setTargetPassword(String targetPassword) { - this.targetPassword = targetPassword; - } - - public static void main(String args[]) throws Exception { - if (args.length < 6) - throw new IllegalArgumentException( - "usage: "); - String sourceServer = args[0]; - String sourceUsername = args[1]; - String sourcePassword = args[2]; - String targetServer = args[3]; - String targetUsername = args[4]; - String targetPassword = args[5]; - - EmailMigration emailMigration = new EmailMigration(); - emailMigration.setSourceServer(sourceServer); - emailMigration.setSourceUsername(sourceUsername); - emailMigration.setSourcePassword(sourcePassword); - emailMigration.setTargetServer(targetServer); - emailMigration.setTargetUsername(targetUsername); - emailMigration.setTargetPassword(targetPassword); - - emailMigration.process(); - } -} diff --git a/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailUtils.java b/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailUtils.java deleted file mode 100644 index 69655471b..000000000 --- a/org.argeo.slc.mail/src/org/argeo/slc/mail/EmailUtils.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.argeo.slc.mail; - -import java.util.Date; - -import javax.mail.Address; -import javax.mail.Flags; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.mail.internet.InternetAddress; -import javax.mail.internet.MimeMessage; - -/** Utilities around emails. */ -public class EmailUtils { - public final static String INBOX = "Inbox"; - public final static String INBOX_UPPER_CASE = "INBOX"; - public final static String MESSAGE_ID = "Message-ID"; - - public static String getMessageId(Message msg) { - try { - return msg instanceof MimeMessage ? ((MimeMessage) msg).getMessageID() : ""; - } catch (MessagingException e) { - throw new IllegalStateException("Cannot extract message id from " + msg, e); - } - } - - public static String describe(Message msg) { - try { - return "Message " + msg.getMessageNumber() + " " + msg.getSentDate().toInstant() + " " + getMessageId(msg); - } catch (MessagingException e) { - throw new IllegalStateException("Cannot describe " + msg, e); - } - } - - static void setHeadersFromFlags(MimeMessage msg, Flags flags) { - try { - StringBuilder status = new StringBuilder(); - if (flags.contains(Flags.Flag.SEEN)) - status.append('R'); - if (!flags.contains(Flags.Flag.RECENT)) - status.append('O'); - if (status.length() > 0) - msg.setHeader("Status", status.toString()); - else - msg.removeHeader("Status"); - - boolean sims = false; - String s = msg.getHeader("X-Status", null); - // is it a SIMS 2.0 format X-Status header? - sims = s != null && s.length() == 4 && s.indexOf('$') >= 0; - //status.setLength(0); - if (flags.contains(Flags.Flag.DELETED)) - status.append('D'); - else if (sims) - status.append('$'); - if (flags.contains(Flags.Flag.FLAGGED)) - status.append('F'); - else if (sims) - status.append('$'); - if (flags.contains(Flags.Flag.ANSWERED)) - status.append('A'); - else if (sims) - status.append('$'); - if (flags.contains(Flags.Flag.DRAFT)) - status.append('T'); - else if (sims) - status.append('$'); - if (status.length() > 0) - msg.setHeader("X-Status", status.toString()); - else - msg.removeHeader("X-Status"); - - String[] userFlags = flags.getUserFlags(); - if (userFlags.length > 0) { - status.setLength(0); - for (int i = 0; i < userFlags.length; i++) - status.append(userFlags[i]).append(' '); - status.setLength(status.length() - 1); // smash trailing space - msg.setHeader("X-Keywords", status.toString()); - } - if (flags.contains(Flags.Flag.DELETED)) { - s = msg.getHeader("X-Dt-Delete-Time", null); - if (s == null) - // XXX - should be time - msg.setHeader("X-Dt-Delete-Time", "1"); - } - } catch (MessagingException e) { - // ignore it - } - } - - protected static String getUnixFrom(MimeMessage msg) { - Address[] afrom; - String from; - Date ddate; - String date; - try { - if ((afrom = msg.getFrom()) == null || - !(afrom[0] instanceof InternetAddress) || - (from = ((InternetAddress)afrom[0]).getAddress()) == null) - from = "UNKNOWN"; - if ((ddate = msg.getReceivedDate()) == null || - (ddate = msg.getSentDate()) == null) - ddate = new Date(); - } catch (MessagingException e) { - from = "UNKNOWN"; - ddate = new Date(); - } - date = ddate.toString(); - // date is of the form "Sat Aug 12 02:30:00 PDT 1995" - // need to strip out the timezone - return "From " + from + " " + - date.substring(0, 20) + date.substring(24); - } - - /** Singleton. */ - private EmailUtils() { - } -} diff --git a/org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailMigration.java b/org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailMigration.java new file mode 100644 index 000000000..79798192a --- /dev/null +++ b/org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailMigration.java @@ -0,0 +1,491 @@ +package org.argeo.slc.mail; + +import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; +import static org.argeo.slc.mail.EmailUtils.describe; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.System.Logger; +import java.nio.file.Path; +import java.util.Date; +import java.util.Properties; + +import javax.mail.FetchProfile; +import javax.mail.Folder; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Multipart; +import javax.mail.Session; +import javax.mail.Store; +import javax.mail.internet.MimeBodyPart; +import javax.mail.internet.MimeMessage; +import javax.mail.search.HeaderTerm; + +import com.sun.mail.imap.IMAPFolder; + +/** Migrates emails from one storage to the another one. */ +public class EmailMigration { + private final static Logger logger = System.getLogger(EmailMigration.class.getName()); + +// private String targetBaseDir; + + private String sourceServer; + private String sourceUsername; + private String sourcePassword; + + private String targetServer; + private String targetUsername; + private String targetPassword; + + public void process() throws MessagingException, IOException { +// Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox"); + + Store sourceStore = null; + try { + Properties sourceProperties = System.getProperties(); + sourceProperties.setProperty("mail.store.protocol", "imaps"); + + Session sourceSession = Session.getInstance(sourceProperties, null); + // session.setDebug(true); + sourceStore = sourceSession.getStore("imaps"); + sourceStore.connect(sourceServer, sourceUsername, sourcePassword); + + Folder defaultFolder = sourceStore.getDefaultFolder(); +// migrateFolders(baseDir, defaultFolder); + + // Always start with Inbox +// Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX); +// migrateFolder(baseDir, inboxFolder); + + Properties targetProperties = System.getProperties(); + targetProperties.setProperty("mail.imap.starttls.enable", "true"); + targetProperties.setProperty("mail.imap.auth", "true"); + + Session targetSession = Session.getInstance(targetProperties, null); + // session.setDebug(true); + Store targetStore = targetSession.getStore("imap"); + targetStore.connect(targetServer, targetUsername, targetPassword); + +// Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX); +// logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount()); +// logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount()); + + migrateFolders(defaultFolder, targetStore); + } finally { + if (sourceStore != null) + sourceStore.close(); + + } + } + + protected void migrateFolders(Folder sourceFolder, Store targetStore) throws MessagingException, IOException { + folders: for (Folder folder : sourceFolder.list()) { + String folderName = folder.getName(); + + String folderFullName = folder.getFullName(); + + // GMail specific + if (folderFullName.equals("[Gmail]")) { + migrateFolders(folder, targetStore); + continue folders; + } + if (folderFullName.startsWith("[Gmail]")) { + // Make it configurable + switch (folderName) { + case "All Mail": + case "Important": + case "Spam": + continue folders; + default: + // does nothing + } + folderFullName = folder.getName(); + } + + int messageCount = (folder.getType() & Folder.HOLDS_MESSAGES) != 0 ? folder.getMessageCount() : 0; + boolean hasSubFolders = (folder.getType() & Folder.HOLDS_FOLDERS) != 0 ? folder.list().length != 0 : false; + Folder targetFolder; + if (hasSubFolders) {// has sub-folders + if (messageCount == 0) { + targetFolder = targetStore.getFolder(folderFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_FOLDERS); + logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName()); + } + } else {// also has messages + Folder parentFolder = targetStore.getFolder(folderFullName); + if (!parentFolder.exists()) { + parentFolder.create(Folder.HOLDS_FOLDERS); + logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName()); + } + String miscFullName = folderFullName + "/_Misc"; + targetFolder = targetStore.getFolder(miscFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_MESSAGES); + logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); + } + } + } else {// no sub-folders + if (messageCount == 0) { // empty + logger.log(DEBUG, "Skip empty folder " + folderFullName); + continue folders; + } + targetFolder = targetStore.getFolder(folderFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_MESSAGES); + logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); + } + } + + if (messageCount != 0) { + + targetFolder.open(Folder.READ_WRITE); + try { + long begin = System.currentTimeMillis(); + folder.open(Folder.READ_ONLY); + migrateFolder(folder, targetFolder); + long duration = System.currentTimeMillis() - begin; + logger.log(DEBUG, folderFullName + " - Migration of " + messageCount + " messages took " + + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)"); + } finally { + folder.close(); + targetFolder.close(); + } + } + + // recursive + if (hasSubFolders) { + migrateFolders(folder, targetStore); + } + } + } + +// protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { +// folders: for (Folder folder : sourceFolder.list()) { +// String folderName = folder.getName(); +// +// if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { +// // Make it configurable +// switch (folderName) { +// case "All Mail": +// case "Important": +// continue folders; +// default: +// // doe nothing +// } +// migrateFolderToFs(baseDir, folder); +// } +// if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { +// migrateFoldersToFs(baseDir.resolve(folder.getName()), folder); +// } +// } +// } + +// protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { +// +// String folderName = sourceFolder.getName(); +// sourceFolder.open(Folder.READ_ONLY); +// +// Folder targetFolder = null; +// try { +// int messageCount = sourceFolder.getMessageCount(); +// logger.log(DEBUG, folderName + " - Message count : " + messageCount); +// if (messageCount == 0) +// return; +//// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount()); +// +// boolean saveAsFiles = false; +// +// if (saveAsFiles) { +// Message messages[] = sourceFolder.getMessages(); +// +// for (int i = 0; i < messages.length; ++i) { +//// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":"); +// Message msg = messages[i]; +//// String from = "unknown"; +//// if (msg.getReplyTo().length >= 1) { +//// from = msg.getReplyTo()[0].toString(); +//// } else if (msg.getFrom().length >= 1) { +//// from = msg.getFrom()[0].toString(); +//// } +// String subject = msg.getSubject(); +// Instant sentDate = msg.getSentDate().toInstant(); +//// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")"); +// String fileName = sentDate + " " + subject; +// Path file = baseDir.resolve(fileName); +// savePartsAsFiles(msg.getContent(), file); +// } +// } +// else { +// long begin = System.currentTimeMillis(); +// targetFolder = openMboxTargetFolder(sourceFolder, baseDir); +// migrateFolder(sourceFolder, targetFolder); +// long duration = System.currentTimeMillis() - begin; +// logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000) +// + " s (" + (duration / messageCount) + " ms per message)"); +// } +// } finally { +// sourceFolder.close(); +// if (targetFolder != null) +// targetFolder.close(); +// } +// } + + protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException { + String folderName = targetFolder.getName(); + + int lastSourceNumber; + int currentTargetMessageCount = targetFolder.getMessageCount(); + if (currentTargetMessageCount != 0) { + MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount); + logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage)); + Date lastTargetSent = lastTargetMessage.getReceivedDate(); + Message[] lastSourceMessage = sourceFolder + .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID())); + if (lastSourceMessage.length == 0) + throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID()); + if (lastSourceMessage.length != 1) { + for (Message msg : lastSourceMessage) { + logger.log(ERROR, "Message " + describe(msg)); + + } + throw new IllegalStateException( + lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant()); + } + lastSourceNumber = lastSourceMessage[0].getMessageNumber(); + } else { + lastSourceNumber = 0; + } + logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber); + + int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber; + + FetchProfile fetchProfile = new FetchProfile(); + fetchProfile.add(FetchProfile.Item.FLAGS); + fetchProfile.add(FetchProfile.Item.ENVELOPE); + fetchProfile.add(FetchProfile.Item.CONTENT_INFO); + fetchProfile.add(FetchProfile.Item.SIZE); + if (sourceFolder instanceof IMAPFolder) { + // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder; + fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS); + fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE); + } + + int batchSize = 100; + int batchCount = countToRetrieve / batchSize; + if (countToRetrieve % batchSize != 0) + batchCount = batchCount + 1; + // int batchCount = 2; // for testing + for (int i = 0; i < batchCount; i++) { + long begin = System.currentTimeMillis(); + + int start = lastSourceNumber + i * batchSize + 1; + int end = lastSourceNumber + (i + 1) * batchSize; + if (end >= (lastSourceNumber + countToRetrieve + 1)) + end = lastSourceNumber + countToRetrieve; + Message[] sourceMessages = sourceFolder.getMessages(start, end); + sourceFolder.fetch(sourceMessages, fetchProfile); + // targetFolder.appendMessages(sourceMessages); + // sourceFolder.copyMessages(sourceMessages,targetFolder); + + copyMessages(sourceMessages, targetFolder); +// copyMessagesToMbox(sourceMessages, targetFolder); + + String describeLast = describe(sourceMessages[sourceMessages.length - 1]); + +// if (i % 10 == 9) { + // free memory from fetched messages + sourceFolder.close(); + targetFolder.close(); + + sourceFolder.open(Folder.READ_ONLY); + targetFolder.open(Folder.READ_WRITE); +// logger.log(DEBUG, "Open/close folder in order to free memory"); +// } + + long duration = System.currentTimeMillis() - begin; + logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, " + + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast); + } + + return targetFolder; + } + +// protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException { +// String folderName = sourceFolder.getName(); +// if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE)) +// folderName = EmailUtils.INBOX;// Inbox +// +// Path targetDir = baseDir;// .resolve("mbox"); +// Files.createDirectories(targetDir); +// Path targetPath; +// if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) { +// Path dir = targetDir.resolve(folderName); +// Files.createDirectories(dir); +// targetPath = dir.resolve("_Misc"); +// } else { +// targetPath = targetDir.resolve(folderName); +// } +// if (!Files.exists(targetPath)) +// Files.createFile(targetPath); +// URLName targetUrlName = new URLName("mbox:" + targetPath.toString()); +// Properties targetProperties = new Properties(); +// // targetProperties.setProperty("mail.mime.address.strict", "false"); +// Session targetSession = Session.getDefaultInstance(targetProperties); +// Folder targetFolder = targetSession.getFolder(targetUrlName); +// targetFolder.open(Folder.READ_WRITE); +// +// return targetFolder; +// } + + protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException { + targetFolder.appendMessages(sourceMessages); + } + +// protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder) +// throws MessagingException, IOException { +// Message[] targetMessages = new Message[sourceMessages.length]; +// for (int j = 0; j < sourceMessages.length; j++) { +// MimeMessage sourceMm = (MimeMessage) sourceMessages[j]; +// InternetHeaders ih = new InternetHeaders(); +// for (Enumeration e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) { +// ih.addHeaderLine(e.nextElement()); +// } +// Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt"); +// Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt"); +// Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING); +// +// // we use ISO_8859_1 because it is more robust than US_ASCII with regard to +// // missing characters +// try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1); +// BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) { +// int lineNumber = 0; +// String line = null; +// try { +// while ((line = reader.readLine()) != null) { +// lineNumber++; +// if (line.startsWith("From ")) { +// writer.write(">" + line); +// logger.log(DEBUG, +// "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line); +// } else { +// writer.write(line); +// } +// writer.newLine(); +// } +// } catch (IOException e) { +// logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource); +// throw e; +// } +// } +// +// MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih, +// new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(), +// EmailUtils.getUnixFrom(sourceMm), true); +// targetMessages[j] = mboxMessage; +// +// // clean up +// Files.delete(tmpFileSource); +// Files.delete(tmpFileTarget); +// } +// targetFolder.appendMessages(targetMessages); +// +// } + + /** Save body parts and attachments as plain files. */ + protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException { + OutputStream out = null; + InputStream in = null; + try { + if (content instanceof Multipart) { + Multipart multi = ((Multipart) content); + int parts = multi.getCount(); + for (int j = 0; j < parts; ++j) { + MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j); + if (part.getContent() instanceof Multipart) { + // part-within-a-part, do some recursion... + savePartsAsFiles(part.getContent(), fileBase); + } else { + String extension = ""; + if (part.isMimeType("text/html")) { + extension = "html"; + } else { + if (part.isMimeType("text/plain")) { + extension = "txt"; + } else { + // Try to get the name of the attachment + extension = part.getDataHandler().getName(); + } + } + String filename = fileBase + "." + extension; + System.out.println("... " + filename); + out = new FileOutputStream(new File(filename)); + in = part.getInputStream(); + int k; + while ((k = in.read()) != -1) { + out.write(k); + } + } + } + } + } finally { + if (in != null) { + in.close(); + } + if (out != null) { + out.flush(); + out.close(); + } + } + } + + public void setSourceServer(String sourceServer) { + this.sourceServer = sourceServer; + } + + public void setSourceUsername(String sourceUsername) { + this.sourceUsername = sourceUsername; + } + + public void setSourcePassword(String sourcePassword) { + this.sourcePassword = sourcePassword; + } + + public void setTargetServer(String targetServer) { + this.targetServer = targetServer; + } + + public void setTargetUsername(String targetUsername) { + this.targetUsername = targetUsername; + } + + public void setTargetPassword(String targetPassword) { + this.targetPassword = targetPassword; + } + + public static void main(String args[]) throws Exception { + if (args.length < 6) + throw new IllegalArgumentException( + "usage: "); + String sourceServer = args[0]; + String sourceUsername = args[1]; + String sourcePassword = args[2]; + String targetServer = args[3]; + String targetUsername = args[4]; + String targetPassword = args[5]; + + EmailMigration emailMigration = new EmailMigration(); + emailMigration.setSourceServer(sourceServer); + emailMigration.setSourceUsername(sourceUsername); + emailMigration.setSourcePassword(sourcePassword); + emailMigration.setTargetServer(targetServer); + emailMigration.setTargetUsername(targetUsername); + emailMigration.setTargetPassword(targetPassword); + + emailMigration.process(); + } +} diff --git a/org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailUtils.java b/org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailUtils.java new file mode 100644 index 000000000..69655471b --- /dev/null +++ b/org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailUtils.java @@ -0,0 +1,118 @@ +package org.argeo.slc.mail; + +import java.util.Date; + +import javax.mail.Address; +import javax.mail.Flags; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +/** Utilities around emails. */ +public class EmailUtils { + public final static String INBOX = "Inbox"; + public final static String INBOX_UPPER_CASE = "INBOX"; + public final static String MESSAGE_ID = "Message-ID"; + + public static String getMessageId(Message msg) { + try { + return msg instanceof MimeMessage ? ((MimeMessage) msg).getMessageID() : ""; + } catch (MessagingException e) { + throw new IllegalStateException("Cannot extract message id from " + msg, e); + } + } + + public static String describe(Message msg) { + try { + return "Message " + msg.getMessageNumber() + " " + msg.getSentDate().toInstant() + " " + getMessageId(msg); + } catch (MessagingException e) { + throw new IllegalStateException("Cannot describe " + msg, e); + } + } + + static void setHeadersFromFlags(MimeMessage msg, Flags flags) { + try { + StringBuilder status = new StringBuilder(); + if (flags.contains(Flags.Flag.SEEN)) + status.append('R'); + if (!flags.contains(Flags.Flag.RECENT)) + status.append('O'); + if (status.length() > 0) + msg.setHeader("Status", status.toString()); + else + msg.removeHeader("Status"); + + boolean sims = false; + String s = msg.getHeader("X-Status", null); + // is it a SIMS 2.0 format X-Status header? + sims = s != null && s.length() == 4 && s.indexOf('$') >= 0; + //status.setLength(0); + if (flags.contains(Flags.Flag.DELETED)) + status.append('D'); + else if (sims) + status.append('$'); + if (flags.contains(Flags.Flag.FLAGGED)) + status.append('F'); + else if (sims) + status.append('$'); + if (flags.contains(Flags.Flag.ANSWERED)) + status.append('A'); + else if (sims) + status.append('$'); + if (flags.contains(Flags.Flag.DRAFT)) + status.append('T'); + else if (sims) + status.append('$'); + if (status.length() > 0) + msg.setHeader("X-Status", status.toString()); + else + msg.removeHeader("X-Status"); + + String[] userFlags = flags.getUserFlags(); + if (userFlags.length > 0) { + status.setLength(0); + for (int i = 0; i < userFlags.length; i++) + status.append(userFlags[i]).append(' '); + status.setLength(status.length() - 1); // smash trailing space + msg.setHeader("X-Keywords", status.toString()); + } + if (flags.contains(Flags.Flag.DELETED)) { + s = msg.getHeader("X-Dt-Delete-Time", null); + if (s == null) + // XXX - should be time + msg.setHeader("X-Dt-Delete-Time", "1"); + } + } catch (MessagingException e) { + // ignore it + } + } + + protected static String getUnixFrom(MimeMessage msg) { + Address[] afrom; + String from; + Date ddate; + String date; + try { + if ((afrom = msg.getFrom()) == null || + !(afrom[0] instanceof InternetAddress) || + (from = ((InternetAddress)afrom[0]).getAddress()) == null) + from = "UNKNOWN"; + if ((ddate = msg.getReceivedDate()) == null || + (ddate = msg.getSentDate()) == null) + ddate = new Date(); + } catch (MessagingException e) { + from = "UNKNOWN"; + ddate = new Date(); + } + date = ddate.toString(); + // date is of the form "Sat Aug 12 02:30:00 PDT 1995" + // need to strip out the timezone + return "From " + from + " " + + date.substring(0, 20) + date.substring(24); + } + + /** Singleton. */ + private EmailUtils() { + } +}