org.argeo.slc.mail/src/org/argeo/slc/mail/EmailMigration.java

   1 package org.argeo.slc.mail;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static org.argeo.slc.mail.EmailUtils.describe;
   6
   7 import java.io.BufferedReader;
   8 import java.io.BufferedWriter;
   9 import java.io.File;
  10 import java.io.FileOutputStream;
  11 import java.io.IOException;
  12 import java.io.InputStream;
  13 import java.io.OutputStream;
  14 import java.lang.System.Logger;
  15 import java.nio.charset.StandardCharsets;
  16 import java.nio.file.Files;
  17 import java.nio.file.Path;
  18 import java.nio.file.Paths;
  19 import java.nio.file.StandardCopyOption;
  20 import java.time.Instant;
  21 import java.util.Date;
  22 import java.util.Enumeration;
  23 import java.util.Properties;
  24
  25 import javax.mail.FetchProfile;
  26 import javax.mail.Folder;
  27 import javax.mail.Message;
  28 import javax.mail.MessagingException;
  29 import javax.mail.Multipart;
  30 import javax.mail.Session;
  31 import javax.mail.Store;
  32 import javax.mail.URLName;
  33 import javax.mail.internet.InternetHeaders;
  34 import javax.mail.internet.MimeBodyPart;
  35 import javax.mail.internet.MimeMessage;
  36 import javax.mail.search.HeaderTerm;
  37 import javax.mail.util.SharedFileInputStream;
  38
  39 import com.sun.mail.imap.IMAPFolder;
  40 import com.sun.mail.mbox.MboxFolder;
  41 import com.sun.mail.mbox.MboxMessage;
  42
  43 /** Migrates emails from one storage to the another one. */
  44 public class EmailMigration {
  45         private final static Logger logger = System.getLogger(EmailMigration.class.getName());
  46
  47         private String targetBaseDir;
  48         private String sourceServer;
  49         private String sourceUsername;
  50         private String sourcePassword;
  51
  52         public void process() throws MessagingException, IOException {
  53                 Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
  54
  55                 Store sourceStore = null;
  56                 try {
  57                         Properties sourceProperties = System.getProperties();
  58                         sourceProperties.setProperty("mail.store.protocol", "imaps");
  59
  60                         Session sourceSession = Session.getDefaultInstance(sourceProperties, null);
  61                         // session.setDebug(true);
  62                         sourceStore = sourceSession.getStore("imaps");
  63                         sourceStore.connect(sourceServer, sourceUsername, sourcePassword);
  64
  65                         Folder defaultFolder = sourceStore.getDefaultFolder();
  66                         migrateFolders(baseDir, defaultFolder);
  67
  68                         // Always start with Inbox
  69 //                      Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
  70 //                      migrateFolder(baseDir, inboxFolder);
  71                 } finally {
  72                         if (sourceStore != null)
  73                                 sourceStore.close();
  74
  75                 }
  76         }
  77
  78         protected void migrateFolders(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
  79                 folders: for (Folder folder : sourceFolder.list()) {
  80                         String folderName = folder.getName();
  81
  82                         if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
  83                                 // Make it configurable
  84                                 switch (folderName) {
  85                                 case "All Mail":
  86                                 case "Important":
  87                                         continue folders;
  88                                 default:
  89                                         // doe nothing
  90                                 }
  91                                 migrateFolder(baseDir, folder);
  92                         }
  93                         if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
  94                                 migrateFolders(baseDir.resolve(folder.getName()), folder);
  95                         }
  96                 }
  97         }
  98
  99         protected void migrateFolder(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 100
 101                 String folderName = sourceFolder.getName();
 102                 sourceFolder.open(Folder.READ_ONLY);
 103
 104                 Folder targetFolder = null;
 105                 try {
 106                         int messageCount = sourceFolder.getMessageCount();
 107                         logger.log(DEBUG, folderName + " - Message count : " + messageCount);
 108                         if (messageCount == 0)
 109                                 return;
 110 //                      logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
 111
 112                         boolean saveAsFiles = false;
 113
 114                         if (saveAsFiles) {
 115                                 Message messages[] = sourceFolder.getMessages();
 116
 117                                 for (int i = 0; i < messages.length; ++i) {
 118 //                                      logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
 119                                         Message msg = messages[i];
 120 //                                      String from = "unknown";
 121 //                                      if (msg.getReplyTo().length >= 1) {
 122 //                                              from = msg.getReplyTo()[0].toString();
 123 //                                      } else if (msg.getFrom().length >= 1) {
 124 //                                              from = msg.getFrom()[0].toString();
 125 //                                      }
 126                                         String subject = msg.getSubject();
 127                                         Instant sentDate = msg.getSentDate().toInstant();
 128 //                                      logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
 129                                         String fileName = sentDate + "  " + subject;
 130                                         Path file = baseDir.resolve(fileName);
 131                                         savePartsAsFiles(msg.getContent(), file);
 132                                 }
 133                         } else {
 134                                 long begin = System.currentTimeMillis();
 135                                 targetFolder = migrateFolderToMbox(baseDir, sourceFolder);
 136                                 long duration = System.currentTimeMillis() - begin;
 137                                 logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000)
 138                                                 + " s (" + (duration / messageCount) + " ms per message)");
 139                         }
 140                 } finally {
 141                         sourceFolder.close();
 142                         if (targetFolder != null)
 143                                 targetFolder.close();
 144                 }
 145         }
 146
 147         protected Folder migrateFolderToMbox(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 148                 String folderName = sourceFolder.getName();
 149                 if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE))
 150                         folderName = EmailUtils.INBOX;// Inbox
 151
 152                 Path targetDir = baseDir;// .resolve("mbox");
 153                 Files.createDirectories(targetDir);
 154                 Path targetPath;
 155                 if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) {
 156                         Path dir = targetDir.resolve(folderName);
 157                         Files.createDirectories(dir);
 158                         targetPath = dir.resolve("_Misc");
 159                 } else {
 160                         targetPath = targetDir.resolve(folderName);
 161                 }
 162                 if (!Files.exists(targetPath))
 163                         Files.createFile(targetPath);
 164                 URLName targetUrlName = new URLName("mbox:" + targetPath.toString());
 165                 Properties targetProperties = new Properties();
 166                 // targetProperties.setProperty("mail.mime.address.strict", "false");
 167                 Session targetSession = Session.getDefaultInstance(targetProperties);
 168                 Folder targetFolder = targetSession.getFolder(targetUrlName);
 169                 targetFolder.open(Folder.READ_WRITE);
 170
 171                 int lastSourceNumber;
 172                 int currentTargetMessageCount = targetFolder.getMessageCount();
 173                 if (currentTargetMessageCount != 0) {
 174                         MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount);
 175                         logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage));
 176                         Date lastTargetSent = lastTargetMessage.getReceivedDate();
 177                         Message[] lastSourceMessage = sourceFolder
 178                                         .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID()));
 179                         if (lastSourceMessage.length == 0)
 180                                 throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID());
 181                         if (lastSourceMessage.length != 1) {
 182                                 for (Message msg : lastSourceMessage) {
 183                                         logger.log(ERROR, "Message " + describe(msg));
 184
 185                                 }
 186                                 throw new IllegalStateException(
 187                                                 lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant());
 188                         }
 189                         lastSourceNumber = lastSourceMessage[0].getMessageNumber();
 190                 } else {
 191                         lastSourceNumber = 0;
 192                 }
 193                 logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber);
 194
 195                 int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber;
 196 //      for (int i = startNumber; i < messageCount; i++) {
 197 //              long begin = System.currentTimeMillis();
 198 //              Message message = sourceFolder.getMessage(i);
 199 //              targetFolder.appendMessages(new Message[] { message });
 200 //              long duration = System.currentTimeMillis() - begin;
 201 //              logger.log(DEBUG, "Message " + i + " migrated in " + duration + " ms");
 202 //      }
 203
 204                 FetchProfile fetchProfile = new FetchProfile();
 205                 fetchProfile.add(FetchProfile.Item.FLAGS);
 206                 fetchProfile.add(FetchProfile.Item.ENVELOPE);
 207                 fetchProfile.add(FetchProfile.Item.CONTENT_INFO);
 208                 fetchProfile.add(FetchProfile.Item.SIZE);
 209                 if (sourceFolder instanceof IMAPFolder) {
 210                         // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
 211                         fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS);
 212                         fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE);
 213                 }
 214
 215                 int batchSize = 100;
 216                 int batchCount = countToRetrieve / batchSize;
 217                 if (countToRetrieve % batchSize != 0)
 218                         batchCount = batchCount + 1;
 219                 // int batchCount = 2; // for testing
 220                 for (int i = 0; i < batchCount; i++) {
 221                         long begin = System.currentTimeMillis();
 222
 223                         int start = lastSourceNumber + i * batchSize + 1;
 224                         int end = lastSourceNumber + (i + 1) * batchSize;
 225                         if (end >= (lastSourceNumber + countToRetrieve + 1))
 226                                 end = lastSourceNumber + countToRetrieve;
 227                         Message[] sourceMessages = sourceFolder.getMessages(start, end);
 228                         sourceFolder.fetch(sourceMessages, fetchProfile);
 229                         // targetFolder.appendMessages(sourceMessages);
 230                         // sourceFolder.copyMessages(sourceMessages,targetFolder);
 231
 232                         Message[] targetMessages = new Message[sourceMessages.length];
 233                         for (int j = 0; j < sourceMessages.length; j++) {
 234                                 MimeMessage sourceMm = (MimeMessage) sourceMessages[j];
 235                                 InternetHeaders ih = new InternetHeaders();
 236                                 for (Enumeration<String> e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) {
 237                                         ih.addHeaderLine(e.nextElement());
 238                                 }
 239 //                      Flags flags = sourceMm.getFlags();
 240 //                      StringBuilder status = new StringBuilder();
 241 //                      if (flags.contains(Flags.Flag.SEEN))
 242 //                              status.append('R');
 243 //                      if (!flags.contains(Flags.Flag.RECENT))
 244 //                              status.append('O');
 245 //                      if (status.length() > 0 && ih.getHeader("X-Status") == null)
 246 //                              ih.setHeader("X-Status", status.toString());
 247
 248                                 Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt");
 249                                 Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt");
 250                                 // logger.log(DEBUG, "tmpFileSource " + tmpFileSource + ", tmpFileTarget " +
 251                                 // tmpFileTarget);
 252                                 Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING);
 253
 254                                 // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
 255                                 // missing characters
 256                                 try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1);
 257                                                 BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) {
 258                                         int lineNumber = 0;
 259                                         String line = null;
 260                                         try {
 261                                                 while ((line = reader.readLine()) != null) {
 262                                                         lineNumber++;
 263                                                         if (line.startsWith("From ")) {
 264                                                                 writer.write(">" + line);
 265                                                                 logger.log(DEBUG, "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm)
 266                                                                                 + ": " + line);
 267                                                         } else {
 268                                                                 writer.write(line);
 269                                                         }
 270                                                         writer.newLine();
 271                                                 }
 272                                         } catch (IOException e) {
 273                                                 logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource);
 274                                                 throw e;
 275                                         }
 276                                 }
 277
 278                                 MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih,
 279                                                 new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(),
 280                                                 EmailUtils.getUnixFrom(sourceMm), true);
 281                                 targetMessages[j] = mboxMessage;
 282
 283                                 // clean up
 284                                 Files.delete(tmpFileSource);
 285                                 Files.delete(tmpFileTarget);
 286                         }
 287                         targetFolder.appendMessages(targetMessages);
 288 //              Message[] targetMessages = targetFolder.getMessages(start, end);
 289 //              for (int j = 0; j < sourceMessages.length; j++) {
 290 //                      EmailUtils.setHeadersFromFlags((MimeMessage) targetMessages[j], sourceMessages[j].getFlags());
 291 ////                    Flags flags = sourceMessages[j].getFlags();
 292 ////                    targetMessages[j].setFlags(flags, true);
 293 //                      targetMessages[j].saveChanges();
 294 //              }
 295
 296                         String describeLast = describe(sourceMessages[sourceMessages.length - 1]);
 297
 298 //              if (i % 10 == 9) {
 299                         // free memory from fetched messages
 300                         sourceFolder.close();
 301                         targetFolder.close();
 302
 303                         sourceFolder.open(Folder.READ_ONLY);
 304                         targetFolder.open(Folder.READ_WRITE);
 305 //                      logger.log(DEBUG, "Open/close folder in order to free memory");
 306 //              }
 307
 308                         long duration = System.currentTimeMillis() - begin;
 309                         logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, "
 310                                         + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast);
 311                 }
 312
 313                 return targetFolder;
 314         }
 315
 316         /** Save body parts and attachments as plain files. */
 317         protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException {
 318                 OutputStream out = null;
 319                 InputStream in = null;
 320                 try {
 321                         if (content instanceof Multipart) {
 322                                 Multipart multi = ((Multipart) content);
 323                                 int parts = multi.getCount();
 324                                 for (int j = 0; j < parts; ++j) {
 325                                         MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j);
 326                                         if (part.getContent() instanceof Multipart) {
 327                                                 // part-within-a-part, do some recursion...
 328                                                 savePartsAsFiles(part.getContent(), fileBase);
 329                                         } else {
 330                                                 String extension = "";
 331                                                 if (part.isMimeType("text/html")) {
 332                                                         extension = "html";
 333                                                 } else {
 334                                                         if (part.isMimeType("text/plain")) {
 335                                                                 extension = "txt";
 336                                                         } else {
 337                                                                 // Try to get the name of the attachment
 338                                                                 extension = part.getDataHandler().getName();
 339                                                         }
 340                                                 }
 341                                                 String filename = fileBase + "." + extension;
 342                                                 System.out.println("... " + filename);
 343                                                 out = new FileOutputStream(new File(filename));
 344                                                 in = part.getInputStream();
 345                                                 int k;
 346                                                 while ((k = in.read()) != -1) {
 347                                                         out.write(k);
 348                                                 }
 349                                         }
 350                                 }
 351                         }
 352                 } finally {
 353                         if (in != null) {
 354                                 in.close();
 355                         }
 356                         if (out != null) {
 357                                 out.flush();
 358                                 out.close();
 359                         }
 360                 }
 361         }
 362
 363         public void setTargetBaseDir(String targetBaseDir) {
 364                 this.targetBaseDir = targetBaseDir;
 365         }
 366
 367         public void setSourceServer(String sourceServer) {
 368                 this.sourceServer = sourceServer;
 369         }
 370
 371         public void setSourceUsername(String sourceUsername) {
 372                 this.sourceUsername = sourceUsername;
 373         }
 374
 375         public void setSourcePassword(String sourcePassword) {
 376                 this.sourcePassword = sourcePassword;
 377         }
 378
 379         public static void main(String args[]) throws Exception {
 380                 if (args.length < 4)
 381                         throw new IllegalArgumentException(
 382                                         "usage: <target base dir> <source IMAP server> <source username> <source password>");
 383                 String targetBaseDir = args[0];
 384                 String sourceServer = args[1];
 385                 String sourceUsername = args[2];
 386                 String sourcePassword = args[3];
 387
 388                 EmailMigration emailMigration = new EmailMigration();
 389                 emailMigration.setTargetBaseDir(targetBaseDir);
 390                 emailMigration.setSourceServer(sourceServer);
 391                 emailMigration.setSourceUsername(sourceUsername);
 392                 emailMigration.setSourcePassword(sourcePassword);
 393
 394                 emailMigration.process();
 395         }
 396 }