From: Mathieu Baudier Date: Mon, 16 May 2022 05:21:39 +0000 (+0200) Subject: Integrate Email migration X-Git-Tag: v2.3.8~108 X-Git-Url: https://git.argeo.org/?p=gpl%2Fargeo-suite.git;a=commitdiff_plain;h=b76993554efeb10d1f7994efdb78403513707f47 Integrate Email migration --- diff --git a/org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java b/org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java new file mode 100644 index 0000000..c5b3083 --- /dev/null +++ b/org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java @@ -0,0 +1,503 @@ +package org.argeo.app.mail; + +import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; +import static org.argeo.app.mail.EmailUtils.describe; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.System.Logger; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Date; +import java.util.Enumeration; +import java.util.Properties; + +import javax.mail.FetchProfile; +import javax.mail.Folder; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.Multipart; +import javax.mail.Session; +import javax.mail.Store; +import javax.mail.URLName; +import javax.mail.internet.InternetHeaders; +import javax.mail.internet.MimeBodyPart; +import javax.mail.internet.MimeMessage; +import javax.mail.search.HeaderTerm; +import javax.mail.util.SharedFileInputStream; + +import com.sun.mail.imap.IMAPFolder; +import com.sun.mail.mbox.MboxFolder; +import com.sun.mail.mbox.MboxMessage; + +/** Migrates emails from one storage to the another one. */ +public class EmailMigration { + private final static Logger logger = System.getLogger(EmailMigration.class.getName()); + +// private String targetBaseDir; + + private String sourceServer; + private String sourceUsername; + private String sourcePassword; + + private String targetServer; + private String targetUsername; + private String targetPassword; + + public void process() throws MessagingException, IOException { +// Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox"); + + Store sourceStore = null; + try { + Properties sourceProperties = System.getProperties(); + sourceProperties.setProperty("mail.store.protocol", "imaps"); + + Session sourceSession = Session.getInstance(sourceProperties, null); + // session.setDebug(true); + sourceStore = sourceSession.getStore("imaps"); + sourceStore.connect(sourceServer, sourceUsername, sourcePassword); + + Folder defaultFolder = sourceStore.getDefaultFolder(); +// migrateFolders(baseDir, defaultFolder); + + // Always start with Inbox +// Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX); +// migrateFolder(baseDir, inboxFolder); + + Properties targetProperties = System.getProperties(); + targetProperties.setProperty("mail.imap.starttls.enable", "true"); + targetProperties.setProperty("mail.imap.auth", "true"); + + Session targetSession = Session.getInstance(targetProperties, null); + // session.setDebug(true); + Store targetStore = targetSession.getStore("imap"); + targetStore.connect(targetServer, targetUsername, targetPassword); + +// Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX); +// logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount()); +// logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount()); + + migrateFolders(defaultFolder, targetStore); + } finally { + if (sourceStore != null) + sourceStore.close(); + + } + } + + protected void migrateFolders(Folder sourceFolder, Store targetStore) throws MessagingException, IOException { + folders: for (Folder folder : sourceFolder.list()) { + String folderName = folder.getName(); + + String folderFullName = folder.getFullName(); + + // GMail specific + if (folderFullName.equals("[Gmail]")) { + migrateFolders(folder, targetStore); + continue folders; + } + if (folderFullName.startsWith("[Gmail]")) { + // Make it configurable + switch (folderName) { + case "All Mail": + case "Important": + case "Spam": + continue folders; + default: + // does nothing + } + folderFullName = folder.getName(); + } + + int messageCount = (folder.getType() & Folder.HOLDS_MESSAGES) != 0 ? folder.getMessageCount() : 0; + boolean hasSubFolders = (folder.getType() & Folder.HOLDS_FOLDERS) != 0 ? folder.list().length != 0 : false; + Folder targetFolder; + if (hasSubFolders) {// has sub-folders + if (messageCount == 0) { + targetFolder = targetStore.getFolder(folderFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_FOLDERS); + logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName()); + } + } else {// also has messages + Folder parentFolder = targetStore.getFolder(folderFullName); + if (!parentFolder.exists()) { + parentFolder.create(Folder.HOLDS_FOLDERS); + logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName()); + } + String miscFullName = folderFullName + "/_Misc"; + targetFolder = targetStore.getFolder(miscFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_MESSAGES); + logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); + } + } + } else {// no sub-folders + if (messageCount == 0) { // empty + logger.log(DEBUG, "Skip empty folder " + folderFullName); + continue folders; + } + targetFolder = targetStore.getFolder(folderFullName); + if (!targetFolder.exists()) { + targetFolder.create(Folder.HOLDS_MESSAGES); + logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName()); + } + } + + if (messageCount != 0) { + + targetFolder.open(Folder.READ_WRITE); + try { + long begin = System.currentTimeMillis(); + folder.open(Folder.READ_ONLY); + migrateFolder(folder, targetFolder); + long duration = System.currentTimeMillis() - begin; + logger.log(DEBUG, folderFullName + " - Migration of " + messageCount + " messages took " + + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)"); + } finally { + folder.close(); + targetFolder.close(); + } + } + + // recursive + if (hasSubFolders) { + migrateFolders(folder, targetStore); + } + } + } + + protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { + folders: for (Folder folder : sourceFolder.list()) { + String folderName = folder.getName(); + + if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) { + // Make it configurable + switch (folderName) { + case "All Mail": + case "Important": + continue folders; + default: + // doe nothing + } + migrateFolderToFs(baseDir, folder); + } + if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) { + migrateFoldersToFs(baseDir.resolve(folder.getName()), folder); + } + } + } + + protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException { + + String folderName = sourceFolder.getName(); + sourceFolder.open(Folder.READ_ONLY); + + Folder targetFolder = null; + try { + int messageCount = sourceFolder.getMessageCount(); + logger.log(DEBUG, folderName + " - Message count : " + messageCount); + if (messageCount == 0) + return; +// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount()); + + boolean saveAsFiles = false; + + if (saveAsFiles) { + Message messages[] = sourceFolder.getMessages(); + + for (int i = 0; i < messages.length; ++i) { +// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":"); + Message msg = messages[i]; +// String from = "unknown"; +// if (msg.getReplyTo().length >= 1) { +// from = msg.getReplyTo()[0].toString(); +// } else if (msg.getFrom().length >= 1) { +// from = msg.getFrom()[0].toString(); +// } + String subject = msg.getSubject(); + Instant sentDate = msg.getSentDate().toInstant(); +// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")"); + String fileName = sentDate + " " + subject; + Path file = baseDir.resolve(fileName); + savePartsAsFiles(msg.getContent(), file); + } + } + else { + long begin = System.currentTimeMillis(); + targetFolder = openMboxTargetFolder(sourceFolder, baseDir); + migrateFolder(sourceFolder, targetFolder); + long duration = System.currentTimeMillis() - begin; + logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000) + + " s (" + (duration / messageCount) + " ms per message)"); + } + } finally { + sourceFolder.close(); + if (targetFolder != null) + targetFolder.close(); + } + } + + protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException { + String folderName = targetFolder.getName(); + + int lastSourceNumber; + int currentTargetMessageCount = targetFolder.getMessageCount(); + if (currentTargetMessageCount != 0) { + MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount); + logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage)); + Date lastTargetSent = lastTargetMessage.getReceivedDate(); + Message[] lastSourceMessage = sourceFolder + .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID())); + if (lastSourceMessage.length == 0) + throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID()); + if (lastSourceMessage.length != 1) { + for (Message msg : lastSourceMessage) { + logger.log(ERROR, "Message " + describe(msg)); + + } + throw new IllegalStateException( + lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant()); + } + lastSourceNumber = lastSourceMessage[0].getMessageNumber(); + } else { + lastSourceNumber = 0; + } + logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber); + + int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber; + + FetchProfile fetchProfile = new FetchProfile(); + fetchProfile.add(FetchProfile.Item.FLAGS); + fetchProfile.add(FetchProfile.Item.ENVELOPE); + fetchProfile.add(FetchProfile.Item.CONTENT_INFO); + fetchProfile.add(FetchProfile.Item.SIZE); + if (sourceFolder instanceof IMAPFolder) { + // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder; + fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS); + fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE); + } + + int batchSize = 100; + int batchCount = countToRetrieve / batchSize; + if (countToRetrieve % batchSize != 0) + batchCount = batchCount + 1; + // int batchCount = 2; // for testing + for (int i = 0; i < batchCount; i++) { + long begin = System.currentTimeMillis(); + + int start = lastSourceNumber + i * batchSize + 1; + int end = lastSourceNumber + (i + 1) * batchSize; + if (end >= (lastSourceNumber + countToRetrieve + 1)) + end = lastSourceNumber + countToRetrieve; + Message[] sourceMessages = sourceFolder.getMessages(start, end); + sourceFolder.fetch(sourceMessages, fetchProfile); + // targetFolder.appendMessages(sourceMessages); + // sourceFolder.copyMessages(sourceMessages,targetFolder); + + copyMessages(sourceMessages, targetFolder); +// copyMessagesToMbox(sourceMessages, targetFolder); + + String describeLast = describe(sourceMessages[sourceMessages.length - 1]); + +// if (i % 10 == 9) { + // free memory from fetched messages + sourceFolder.close(); + targetFolder.close(); + + sourceFolder.open(Folder.READ_ONLY); + targetFolder.open(Folder.READ_WRITE); +// logger.log(DEBUG, "Open/close folder in order to free memory"); +// } + + long duration = System.currentTimeMillis() - begin; + logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, " + + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast); + } + + return targetFolder; + } + + protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException { + String folderName = sourceFolder.getName(); + if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE)) + folderName = EmailUtils.INBOX;// Inbox + + Path targetDir = baseDir;// .resolve("mbox"); + Files.createDirectories(targetDir); + Path targetPath; + if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) { + Path dir = targetDir.resolve(folderName); + Files.createDirectories(dir); + targetPath = dir.resolve("_Misc"); + } else { + targetPath = targetDir.resolve(folderName); + } + if (!Files.exists(targetPath)) + Files.createFile(targetPath); + URLName targetUrlName = new URLName("mbox:" + targetPath.toString()); + Properties targetProperties = new Properties(); + // targetProperties.setProperty("mail.mime.address.strict", "false"); + Session targetSession = Session.getDefaultInstance(targetProperties); + Folder targetFolder = targetSession.getFolder(targetUrlName); + targetFolder.open(Folder.READ_WRITE); + + return targetFolder; + } + + protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException { + targetFolder.appendMessages(sourceMessages); + } + + protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder) + throws MessagingException, IOException { + Message[] targetMessages = new Message[sourceMessages.length]; + for (int j = 0; j < sourceMessages.length; j++) { + MimeMessage sourceMm = (MimeMessage) sourceMessages[j]; + InternetHeaders ih = new InternetHeaders(); + for (Enumeration e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) { + ih.addHeaderLine(e.nextElement()); + } + Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt"); + Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt"); + Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING); + + // we use ISO_8859_1 because it is more robust than US_ASCII with regard to + // missing characters + try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1); + BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) { + int lineNumber = 0; + String line = null; + try { + while ((line = reader.readLine()) != null) { + lineNumber++; + if (line.startsWith("From ")) { + writer.write(">" + line); + logger.log(DEBUG, + "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line); + } else { + writer.write(line); + } + writer.newLine(); + } + } catch (IOException e) { + logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource); + throw e; + } + } + + MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih, + new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(), + EmailUtils.getUnixFrom(sourceMm), true); + targetMessages[j] = mboxMessage; + + // clean up + Files.delete(tmpFileSource); + Files.delete(tmpFileTarget); + } + targetFolder.appendMessages(targetMessages); + + } + + /** Save body parts and attachments as plain files. */ + protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException { + OutputStream out = null; + InputStream in = null; + try { + if (content instanceof Multipart) { + Multipart multi = ((Multipart) content); + int parts = multi.getCount(); + for (int j = 0; j < parts; ++j) { + MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j); + if (part.getContent() instanceof Multipart) { + // part-within-a-part, do some recursion... + savePartsAsFiles(part.getContent(), fileBase); + } else { + String extension = ""; + if (part.isMimeType("text/html")) { + extension = "html"; + } else { + if (part.isMimeType("text/plain")) { + extension = "txt"; + } else { + // Try to get the name of the attachment + extension = part.getDataHandler().getName(); + } + } + String filename = fileBase + "." + extension; + System.out.println("... " + filename); + out = new FileOutputStream(new File(filename)); + in = part.getInputStream(); + int k; + while ((k = in.read()) != -1) { + out.write(k); + } + } + } + } + } finally { + if (in != null) { + in.close(); + } + if (out != null) { + out.flush(); + out.close(); + } + } + } + + public void setSourceServer(String sourceServer) { + this.sourceServer = sourceServer; + } + + public void setSourceUsername(String sourceUsername) { + this.sourceUsername = sourceUsername; + } + + public void setSourcePassword(String sourcePassword) { + this.sourcePassword = sourcePassword; + } + + public void setTargetServer(String targetServer) { + this.targetServer = targetServer; + } + + public void setTargetUsername(String targetUsername) { + this.targetUsername = targetUsername; + } + + public void setTargetPassword(String targetPassword) { + this.targetPassword = targetPassword; + } + + public static void main(String args[]) throws Exception { + if (args.length < 6) + throw new IllegalArgumentException( + "usage: "); + String sourceServer = args[0]; + String sourceUsername = args[1]; + String sourcePassword = args[2]; + String targetServer = args[3]; + String targetUsername = args[4]; + String targetPassword = args[5]; + + EmailMigration emailMigration = new EmailMigration(); + emailMigration.setSourceServer(sourceServer); + emailMigration.setSourceUsername(sourceUsername); + emailMigration.setSourcePassword(sourcePassword); + emailMigration.setTargetServer(targetServer); + emailMigration.setTargetUsername(targetUsername); + emailMigration.setTargetPassword(targetPassword); + + emailMigration.process(); + } +} diff --git a/org.argeo.app.core/src/org/argeo/app/mail/EmailUtils.java b/org.argeo.app.core/src/org/argeo/app/mail/EmailUtils.java new file mode 100644 index 0000000..694c17c --- /dev/null +++ b/org.argeo.app.core/src/org/argeo/app/mail/EmailUtils.java @@ -0,0 +1,118 @@ +package org.argeo.app.mail; + +import java.util.Date; + +import javax.mail.Address; +import javax.mail.Flags; +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.mail.internet.InternetAddress; +import javax.mail.internet.MimeMessage; + +/** Utilities around emails. */ +public class EmailUtils { + public final static String INBOX = "Inbox"; + public final static String INBOX_UPPER_CASE = "INBOX"; + public final static String MESSAGE_ID = "Message-ID"; + + public static String getMessageId(Message msg) { + try { + return msg instanceof MimeMessage ? ((MimeMessage) msg).getMessageID() : ""; + } catch (MessagingException e) { + throw new IllegalStateException("Cannot extract message id from " + msg, e); + } + } + + public static String describe(Message msg) { + try { + return "Message " + msg.getMessageNumber() + " " + msg.getSentDate().toInstant() + " " + getMessageId(msg); + } catch (MessagingException e) { + throw new IllegalStateException("Cannot describe " + msg, e); + } + } + + static void setHeadersFromFlags(MimeMessage msg, Flags flags) { + try { + StringBuilder status = new StringBuilder(); + if (flags.contains(Flags.Flag.SEEN)) + status.append('R'); + if (!flags.contains(Flags.Flag.RECENT)) + status.append('O'); + if (status.length() > 0) + msg.setHeader("Status", status.toString()); + else + msg.removeHeader("Status"); + + boolean sims = false; + String s = msg.getHeader("X-Status", null); + // is it a SIMS 2.0 format X-Status header? + sims = s != null && s.length() == 4 && s.indexOf('$') >= 0; + //status.setLength(0); + if (flags.contains(Flags.Flag.DELETED)) + status.append('D'); + else if (sims) + status.append('$'); + if (flags.contains(Flags.Flag.FLAGGED)) + status.append('F'); + else if (sims) + status.append('$'); + if (flags.contains(Flags.Flag.ANSWERED)) + status.append('A'); + else if (sims) + status.append('$'); + if (flags.contains(Flags.Flag.DRAFT)) + status.append('T'); + else if (sims) + status.append('$'); + if (status.length() > 0) + msg.setHeader("X-Status", status.toString()); + else + msg.removeHeader("X-Status"); + + String[] userFlags = flags.getUserFlags(); + if (userFlags.length > 0) { + status.setLength(0); + for (int i = 0; i < userFlags.length; i++) + status.append(userFlags[i]).append(' '); + status.setLength(status.length() - 1); // smash trailing space + msg.setHeader("X-Keywords", status.toString()); + } + if (flags.contains(Flags.Flag.DELETED)) { + s = msg.getHeader("X-Dt-Delete-Time", null); + if (s == null) + // XXX - should be time + msg.setHeader("X-Dt-Delete-Time", "1"); + } + } catch (MessagingException e) { + // ignore it + } + } + + protected static String getUnixFrom(MimeMessage msg) { + Address[] afrom; + String from; + Date ddate; + String date; + try { + if ((afrom = msg.getFrom()) == null || + !(afrom[0] instanceof InternetAddress) || + (from = ((InternetAddress)afrom[0]).getAddress()) == null) + from = "UNKNOWN"; + if ((ddate = msg.getReceivedDate()) == null || + (ddate = msg.getSentDate()) == null) + ddate = new Date(); + } catch (MessagingException e) { + from = "UNKNOWN"; + ddate = new Date(); + } + date = ddate.toString(); + // date is of the form "Sat Aug 12 02:30:00 PDT 1995" + // need to strip out the timezone + return "From " + from + " " + + date.substring(0, 20) + date.substring(24); + } + + /** Singleton. */ + private EmailUtils() { + } +}