EmailMigration.java

   1 package org.argeo.slc.mail;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static org.argeo.slc.mail.EmailUtils.describe;
   6
   7 import java.io.File;
   8 import java.io.FileOutputStream;
   9 import java.io.IOException;
  10 import java.io.InputStream;
  11 import java.io.OutputStream;
  12 import java.lang.System.Logger;
  13 import java.nio.file.Path;
  14 import java.util.Date;
  15 import java.util.Properties;
  16
  17 import javax.mail.FetchProfile;
  18 import javax.mail.Folder;
  19 import javax.mail.Message;
  20 import javax.mail.MessagingException;
  21 import javax.mail.Multipart;
  22 import javax.mail.Session;
  23 import javax.mail.Store;
  24 import javax.mail.internet.MimeBodyPart;
  25 import javax.mail.internet.MimeMessage;
  26 import javax.mail.search.HeaderTerm;
  27
  28 import com.sun.mail.imap.IMAPFolder;
  29
  30 /** Migrates emails from one storage to the another one. */
  31 public class EmailMigration {
  32         private final static Logger logger = System.getLogger(EmailMigration.class.getName());
  33
  34 //      private String targetBaseDir;
  35
  36         private String sourceServer;
  37         private String sourceUsername;
  38         private String sourcePassword;
  39
  40         private String targetServer;
  41         private String targetUsername;
  42         private String targetPassword;
  43
  44         public void process() throws MessagingException, IOException {
  45 //              Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
  46
  47                 Store sourceStore = null;
  48                 try {
  49                         Properties sourceProperties = System.getProperties();
  50                         sourceProperties.setProperty("mail.store.protocol", "imaps");
  51
  52                         Session sourceSession = Session.getInstance(sourceProperties, null);
  53                         // session.setDebug(true);
  54                         sourceStore = sourceSession.getStore("imaps");
  55                         sourceStore.connect(sourceServer, sourceUsername, sourcePassword);
  56
  57                         Folder defaultFolder = sourceStore.getDefaultFolder();
  58 //                      migrateFolders(baseDir, defaultFolder);
  59
  60                         // Always start with Inbox
  61 //                      Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
  62 //                      migrateFolder(baseDir, inboxFolder);
  63
  64                         Properties targetProperties = System.getProperties();
  65                         targetProperties.setProperty("mail.imap.starttls.enable", "true");
  66                         targetProperties.setProperty("mail.imap.auth", "true");
  67
  68                         Session targetSession = Session.getInstance(targetProperties, null);
  69                         // session.setDebug(true);
  70                         Store targetStore = targetSession.getStore("imap");
  71                         targetStore.connect(targetServer, targetUsername, targetPassword);
  72
  73 //                      Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX);
  74 //                      logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount());
  75 //                      logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount());
  76
  77                         migrateFolders(defaultFolder, targetStore);
  78                 } finally {
  79                         if (sourceStore != null)
  80                                 sourceStore.close();
  81
  82                 }
  83         }
  84
  85         protected void migrateFolders(Folder sourceFolder, Store targetStore) throws MessagingException, IOException {
  86                 folders: for (Folder folder : sourceFolder.list()) {
  87                         String folderName = folder.getName();
  88
  89                         String folderFullName = folder.getFullName();
  90
  91                         // GMail specific
  92                         if (folderFullName.equals("[Gmail]")) {
  93                                 migrateFolders(folder, targetStore);
  94                                 continue folders;
  95                         }
  96                         if (folderFullName.startsWith("[Gmail]")) {
  97                                 // Make it configurable
  98                                 switch (folderName) {
  99                                 case "All Mail":
 100                                 case "Important":
 101                                 case "Spam":
 102                                         continue folders;
 103                                 default:
 104                                         // does nothing
 105                                 }
 106                                 folderFullName = folder.getName();
 107                         }
 108
 109                         int messageCount = (folder.getType() & Folder.HOLDS_MESSAGES) != 0 ? folder.getMessageCount() : 0;
 110                         boolean hasSubFolders = (folder.getType() & Folder.HOLDS_FOLDERS) != 0 ? folder.list().length != 0 : false;
 111                         Folder targetFolder;
 112                         if (hasSubFolders) {// has sub-folders
 113                                 if (messageCount == 0) {
 114                                         targetFolder = targetStore.getFolder(folderFullName);
 115                                         if (!targetFolder.exists()) {
 116                                                 targetFolder.create(Folder.HOLDS_FOLDERS);
 117                                                 logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName());
 118                                         }
 119                                 } else {// also has messages
 120                                         Folder parentFolder = targetStore.getFolder(folderFullName);
 121                                         if (!parentFolder.exists()) {
 122                                                 parentFolder.create(Folder.HOLDS_FOLDERS);
 123                                                 logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName());
 124                                         }
 125                                         String miscFullName = folderFullName + "/_Misc";
 126                                         targetFolder = targetStore.getFolder(miscFullName);
 127                                         if (!targetFolder.exists()) {
 128                                                 targetFolder.create(Folder.HOLDS_MESSAGES);
 129                                                 logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName());
 130                                         }
 131                                 }
 132                         } else {// no sub-folders
 133                                 if (messageCount == 0) { // empty
 134                                         logger.log(DEBUG, "Skip empty folder " + folderFullName);
 135                                         continue folders;
 136                                 }
 137                                 targetFolder = targetStore.getFolder(folderFullName);
 138                                 if (!targetFolder.exists()) {
 139                                         targetFolder.create(Folder.HOLDS_MESSAGES);
 140                                         logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName());
 141                                 }
 142                         }
 143
 144                         if (messageCount != 0) {
 145
 146                                 targetFolder.open(Folder.READ_WRITE);
 147                                 try {
 148                                         long begin = System.currentTimeMillis();
 149                                         folder.open(Folder.READ_ONLY);
 150                                         migrateFolder(folder, targetFolder);
 151                                         long duration = System.currentTimeMillis() - begin;
 152                                         logger.log(DEBUG, folderFullName + " - Migration of " + messageCount + " messages took "
 153                                                         + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)");
 154                                 } finally {
 155                                         folder.close();
 156                                         targetFolder.close();
 157                                 }
 158                         }
 159
 160                         // recursive
 161                         if (hasSubFolders) {
 162                                 migrateFolders(folder, targetStore);
 163                         }
 164                 }
 165         }
 166
 167 //      protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 168 //              folders: for (Folder folder : sourceFolder.list()) {
 169 //                      String folderName = folder.getName();
 170 //
 171 //                      if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
 172 //                              // Make it configurable
 173 //                              switch (folderName) {
 174 //                              case "All Mail":
 175 //                              case "Important":
 176 //                                      continue folders;
 177 //                              default:
 178 //                                      // doe nothing
 179 //                              }
 180 //                              migrateFolderToFs(baseDir, folder);
 181 //                      }
 182 //                      if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
 183 //                              migrateFoldersToFs(baseDir.resolve(folder.getName()), folder);
 184 //                      }
 185 //              }
 186 //      }
 187
 188 //      protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 189 //
 190 //              String folderName = sourceFolder.getName();
 191 //              sourceFolder.open(Folder.READ_ONLY);
 192 //
 193 //              Folder targetFolder = null;
 194 //              try {
 195 //                      int messageCount = sourceFolder.getMessageCount();
 196 //                      logger.log(DEBUG, folderName + " - Message count : " + messageCount);
 197 //                      if (messageCount == 0)
 198 //                              return;
 199 ////                    logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
 200 //
 201 //                      boolean saveAsFiles = false;
 202 //
 203 //                      if (saveAsFiles) {
 204 //                              Message messages[] = sourceFolder.getMessages();
 205 //
 206 //                              for (int i = 0; i < messages.length; ++i) {
 207 ////                                    logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
 208 //                                      Message msg = messages[i];
 209 ////                                    String from = "unknown";
 210 ////                                    if (msg.getReplyTo().length >= 1) {
 211 ////                                            from = msg.getReplyTo()[0].toString();
 212 ////                                    } else if (msg.getFrom().length >= 1) {
 213 ////                                            from = msg.getFrom()[0].toString();
 214 ////                                    }
 215 //                                      String subject = msg.getSubject();
 216 //                                      Instant sentDate = msg.getSentDate().toInstant();
 217 ////                                    logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
 218 //                                      String fileName = sentDate + "  " + subject;
 219 //                                      Path file = baseDir.resolve(fileName);
 220 //                                      savePartsAsFiles(msg.getContent(), file);
 221 //                              }
 222 //                      }
 223 //                      else {
 224 //                              long begin = System.currentTimeMillis();
 225 //                              targetFolder = openMboxTargetFolder(sourceFolder, baseDir);
 226 //                              migrateFolder(sourceFolder, targetFolder);
 227 //                              long duration = System.currentTimeMillis() - begin;
 228 //                              logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000)
 229 //                                              + " s (" + (duration / messageCount) + " ms per message)");
 230 //                      }
 231 //              } finally {
 232 //                      sourceFolder.close();
 233 //                      if (targetFolder != null)
 234 //                              targetFolder.close();
 235 //              }
 236 //      }
 237
 238         protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException {
 239                 String folderName = targetFolder.getName();
 240
 241                 int lastSourceNumber;
 242                 int currentTargetMessageCount = targetFolder.getMessageCount();
 243                 if (currentTargetMessageCount != 0) {
 244                         MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount);
 245                         logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage));
 246                         Date lastTargetSent = lastTargetMessage.getReceivedDate();
 247                         Message[] lastSourceMessage = sourceFolder
 248                                         .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID()));
 249                         if (lastSourceMessage.length == 0)
 250                                 throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID());
 251                         if (lastSourceMessage.length != 1) {
 252                                 for (Message msg : lastSourceMessage) {
 253                                         logger.log(ERROR, "Message " + describe(msg));
 254
 255                                 }
 256                                 throw new IllegalStateException(
 257                                                 lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant());
 258                         }
 259                         lastSourceNumber = lastSourceMessage[0].getMessageNumber();
 260                 } else {
 261                         lastSourceNumber = 0;
 262                 }
 263                 logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber);
 264
 265                 int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber;
 266
 267                 FetchProfile fetchProfile = new FetchProfile();
 268                 fetchProfile.add(FetchProfile.Item.FLAGS);
 269                 fetchProfile.add(FetchProfile.Item.ENVELOPE);
 270                 fetchProfile.add(FetchProfile.Item.CONTENT_INFO);
 271                 fetchProfile.add(FetchProfile.Item.SIZE);
 272                 if (sourceFolder instanceof IMAPFolder) {
 273                         // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
 274                         fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS);
 275                         fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE);
 276                 }
 277
 278                 int batchSize = 100;
 279                 int batchCount = countToRetrieve / batchSize;
 280                 if (countToRetrieve % batchSize != 0)
 281                         batchCount = batchCount + 1;
 282                 // int batchCount = 2; // for testing
 283                 for (int i = 0; i < batchCount; i++) {
 284                         long begin = System.currentTimeMillis();
 285
 286                         int start = lastSourceNumber + i * batchSize + 1;
 287                         int end = lastSourceNumber + (i + 1) * batchSize;
 288                         if (end >= (lastSourceNumber + countToRetrieve + 1))
 289                                 end = lastSourceNumber + countToRetrieve;
 290                         Message[] sourceMessages = sourceFolder.getMessages(start, end);
 291                         sourceFolder.fetch(sourceMessages, fetchProfile);
 292                         // targetFolder.appendMessages(sourceMessages);
 293                         // sourceFolder.copyMessages(sourceMessages,targetFolder);
 294
 295                         copyMessages(sourceMessages, targetFolder);
 296 //                      copyMessagesToMbox(sourceMessages, targetFolder);
 297
 298                         String describeLast = describe(sourceMessages[sourceMessages.length - 1]);
 299
 300 //              if (i % 10 == 9) {
 301                         // free memory from fetched messages
 302                         sourceFolder.close();
 303                         targetFolder.close();
 304
 305                         sourceFolder.open(Folder.READ_ONLY);
 306                         targetFolder.open(Folder.READ_WRITE);
 307 //                      logger.log(DEBUG, "Open/close folder in order to free memory");
 308 //              }
 309
 310                         long duration = System.currentTimeMillis() - begin;
 311                         logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, "
 312                                         + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast);
 313                 }
 314
 315                 return targetFolder;
 316         }
 317
 318 //      protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException {
 319 //              String folderName = sourceFolder.getName();
 320 //              if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE))
 321 //                      folderName = EmailUtils.INBOX;// Inbox
 322 //
 323 //              Path targetDir = baseDir;// .resolve("mbox");
 324 //              Files.createDirectories(targetDir);
 325 //              Path targetPath;
 326 //              if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) {
 327 //                      Path dir = targetDir.resolve(folderName);
 328 //                      Files.createDirectories(dir);
 329 //                      targetPath = dir.resolve("_Misc");
 330 //              } else {
 331 //                      targetPath = targetDir.resolve(folderName);
 332 //              }
 333 //              if (!Files.exists(targetPath))
 334 //                      Files.createFile(targetPath);
 335 //              URLName targetUrlName = new URLName("mbox:" + targetPath.toString());
 336 //              Properties targetProperties = new Properties();
 337 //              // targetProperties.setProperty("mail.mime.address.strict", "false");
 338 //              Session targetSession = Session.getDefaultInstance(targetProperties);
 339 //              Folder targetFolder = targetSession.getFolder(targetUrlName);
 340 //              targetFolder.open(Folder.READ_WRITE);
 341 //
 342 //              return targetFolder;
 343 //      }
 344
 345         protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException {
 346                 targetFolder.appendMessages(sourceMessages);
 347         }
 348
 349 //      protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder)
 350 //                      throws MessagingException, IOException {
 351 //              Message[] targetMessages = new Message[sourceMessages.length];
 352 //              for (int j = 0; j < sourceMessages.length; j++) {
 353 //                      MimeMessage sourceMm = (MimeMessage) sourceMessages[j];
 354 //                      InternetHeaders ih = new InternetHeaders();
 355 //                      for (Enumeration<String> e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) {
 356 //                              ih.addHeaderLine(e.nextElement());
 357 //                      }
 358 //                      Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt");
 359 //                      Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt");
 360 //                      Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING);
 361 //
 362 //                      // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
 363 //                      // missing characters
 364 //                      try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1);
 365 //                                      BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) {
 366 //                              int lineNumber = 0;
 367 //                              String line = null;
 368 //                              try {
 369 //                                      while ((line = reader.readLine()) != null) {
 370 //                                              lineNumber++;
 371 //                                              if (line.startsWith("From ")) {
 372 //                                                      writer.write(">" + line);
 373 //                                                      logger.log(DEBUG,
 374 //                                                                      "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line);
 375 //                                              } else {
 376 //                                                      writer.write(line);
 377 //                                              }
 378 //                                              writer.newLine();
 379 //                                      }
 380 //                              } catch (IOException e) {
 381 //                                      logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource);
 382 //                                      throw e;
 383 //                              }
 384 //                      }
 385 //
 386 //                      MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih,
 387 //                                      new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(),
 388 //                                      EmailUtils.getUnixFrom(sourceMm), true);
 389 //                      targetMessages[j] = mboxMessage;
 390 //
 391 //                      // clean up
 392 //                      Files.delete(tmpFileSource);
 393 //                      Files.delete(tmpFileTarget);
 394 //              }
 395 //              targetFolder.appendMessages(targetMessages);
 396 //
 397 //      }
 398
 399         /** Save body parts and attachments as plain files. */
 400         protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException {
 401                 OutputStream out = null;
 402                 InputStream in = null;
 403                 try {
 404                         if (content instanceof Multipart) {
 405                                 Multipart multi = ((Multipart) content);
 406                                 int parts = multi.getCount();
 407                                 for (int j = 0; j < parts; ++j) {
 408                                         MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j);
 409                                         if (part.getContent() instanceof Multipart) {
 410                                                 // part-within-a-part, do some recursion...
 411                                                 savePartsAsFiles(part.getContent(), fileBase);
 412                                         } else {
 413                                                 String extension = "";
 414                                                 if (part.isMimeType("text/html")) {
 415                                                         extension = "html";
 416                                                 } else {
 417                                                         if (part.isMimeType("text/plain")) {
 418                                                                 extension = "txt";
 419                                                         } else {
 420                                                                 // Try to get the name of the attachment
 421                                                                 extension = part.getDataHandler().getName();
 422                                                         }
 423                                                 }
 424                                                 String filename = fileBase + "." + extension;
 425                                                 System.out.println("... " + filename);
 426                                                 out = new FileOutputStream(new File(filename));
 427                                                 in = part.getInputStream();
 428                                                 int k;
 429                                                 while ((k = in.read()) != -1) {
 430                                                         out.write(k);
 431                                                 }
 432                                         }
 433                                 }
 434                         }
 435                 } finally {
 436                         if (in != null) {
 437                                 in.close();
 438                         }
 439                         if (out != null) {
 440                                 out.flush();
 441                                 out.close();
 442                         }
 443                 }
 444         }
 445
 446         public void setSourceServer(String sourceServer) {
 447                 this.sourceServer = sourceServer;
 448         }
 449
 450         public void setSourceUsername(String sourceUsername) {
 451                 this.sourceUsername = sourceUsername;
 452         }
 453
 454         public void setSourcePassword(String sourcePassword) {
 455                 this.sourcePassword = sourcePassword;
 456         }
 457
 458         public void setTargetServer(String targetServer) {
 459                 this.targetServer = targetServer;
 460         }
 461
 462         public void setTargetUsername(String targetUsername) {
 463                 this.targetUsername = targetUsername;
 464         }
 465
 466         public void setTargetPassword(String targetPassword) {
 467                 this.targetPassword = targetPassword;
 468         }
 469
 470         public static void main(String args[]) throws Exception {
 471                 if (args.length < 6)
 472                         throw new IllegalArgumentException(
 473                                         "usage: <source IMAP server> <source username> <source password> <target IMAP server> <target username> <target password>");
 474                 String sourceServer = args[0];
 475                 String sourceUsername = args[1];
 476                 String sourcePassword = args[2];
 477                 String targetServer = args[3];
 478                 String targetUsername = args[4];
 479                 String targetPassword = args[5];
 480
 481                 EmailMigration emailMigration = new EmailMigration();
 482                 emailMigration.setSourceServer(sourceServer);
 483                 emailMigration.setSourceUsername(sourceUsername);
 484                 emailMigration.setSourcePassword(sourcePassword);
 485                 emailMigration.setTargetServer(targetServer);
 486                 emailMigration.setTargetUsername(targetUsername);
 487                 emailMigration.setTargetPassword(targetPassword);
 488
 489                 emailMigration.process();
 490         }
 491 }