org.argeo.app.core/src/org/argeo/app/mail/EmailMigration.java

   1 package org.argeo.app.mail;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static org.argeo.app.mail.EmailUtils.describe;
   6
   7 import java.io.BufferedReader;
   8 import java.io.BufferedWriter;
   9 import java.io.File;
  10 import java.io.FileOutputStream;
  11 import java.io.IOException;
  12 import java.io.InputStream;
  13 import java.io.OutputStream;
  14 import java.lang.System.Logger;
  15 import java.nio.charset.StandardCharsets;
  16 import java.nio.file.Files;
  17 import java.nio.file.Path;
  18 import java.nio.file.StandardCopyOption;
  19 import java.time.Instant;
  20 import java.util.Date;
  21 import java.util.Enumeration;
  22 import java.util.Properties;
  23
  24 import javax.mail.FetchProfile;
  25 import javax.mail.Folder;
  26 import javax.mail.Message;
  27 import javax.mail.MessagingException;
  28 import javax.mail.Multipart;
  29 import javax.mail.Session;
  30 import javax.mail.Store;
  31 import javax.mail.URLName;
  32 import javax.mail.internet.InternetHeaders;
  33 import javax.mail.internet.MimeBodyPart;
  34 import javax.mail.internet.MimeMessage;
  35 import javax.mail.search.HeaderTerm;
  36 import javax.mail.util.SharedFileInputStream;
  37
  38 import com.sun.mail.imap.IMAPFolder;
  39 import com.sun.mail.mbox.MboxFolder;
  40 import com.sun.mail.mbox.MboxMessage;
  41
  42 /** Migrates emails from one storage to the another one. */
  43 public class EmailMigration {
  44         private final static Logger logger = System.getLogger(EmailMigration.class.getName());
  45
  46 //      private String targetBaseDir;
  47
  48         private String sourceServer;
  49         private String sourceUsername;
  50         private String sourcePassword;
  51
  52         private String targetServer;
  53         private String targetUsername;
  54         private String targetPassword;
  55
  56         private boolean targetSupportDualTypeFolders = true;
  57
  58         public void process() throws MessagingException, IOException {
  59 //              Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
  60
  61                 Store sourceStore = null;
  62                 try {
  63                         Properties sourceProperties = System.getProperties();
  64                         sourceProperties.setProperty("mail.store.protocol", "imaps");
  65
  66                         Session sourceSession = Session.getInstance(sourceProperties, null);
  67                         // session.setDebug(true);
  68                         sourceStore = sourceSession.getStore("imaps");
  69                         sourceStore.connect(sourceServer, sourceUsername, sourcePassword);
  70
  71                         Folder defaultFolder = sourceStore.getDefaultFolder();
  72 //                      migrateFolders(baseDir, defaultFolder);
  73
  74                         // Always start with Inbox
  75 //                      Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
  76 //                      migrateFolder(baseDir, inboxFolder);
  77
  78                         Properties targetProperties = System.getProperties();
  79                         targetProperties.setProperty("mail.imap.starttls.enable", "true");
  80                         targetProperties.setProperty("mail.imap.auth", "true");
  81
  82                         Session targetSession = Session.getInstance(targetProperties, null);
  83                         // session.setDebug(true);
  84                         Store targetStore = targetSession.getStore("imap");
  85                         targetStore.connect(targetServer, targetUsername, targetPassword);
  86
  87 //                      Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX);
  88 //                      logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount());
  89 //                      logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount());
  90
  91                         migrateFolders(defaultFolder, targetStore);
  92                 } finally {
  93                         if (sourceStore != null)
  94                                 sourceStore.close();
  95
  96                 }
  97         }
  98
  99         protected void migrateFolders(Folder sourceParentFolder, Store targetStore) throws MessagingException, IOException {
 100                 folders: for (Folder sourceFolder : sourceParentFolder.list()) {
 101                         String sourceFolderName = sourceFolder.getName();
 102
 103                         String sourceFolderFullName = sourceFolder.getFullName();
 104                         char sourceFolderSeparator = sourceParentFolder.getSeparator();
 105                         char targetFolderSeparator = targetStore.getDefaultFolder().getSeparator();
 106                         String targetFolderFullName = sourceFolderFullName.replace(sourceFolderSeparator, targetFolderSeparator);
 107
 108                         // GMail specific
 109                         if (sourceFolderFullName.equals("[Gmail]")) {
 110                                 migrateFolders(sourceFolder, targetStore);
 111                                 continue folders;
 112                         }
 113                         if (sourceFolderFullName.startsWith("[Gmail]")) {
 114                                 String subFolderName = null;
 115                                 // Make it configurable
 116                                 switch (sourceFolderName) {
 117                                 case "All Mail":
 118                                 case "Important":
 119                                 case "Spam":
 120                                         continue folders;
 121                                 case "Sent Mail":
 122                                         subFolderName = "Sent";
 123                                 default:
 124                                         // does nothing
 125                                 }
 126                                 targetFolderFullName = subFolderName == null ? sourceFolder.getName() : subFolderName;
 127                         }
 128
 129                         // nature of the source folder
 130                         int messageCount = (sourceFolder.getType() & Folder.HOLDS_MESSAGES) != 0 ? sourceFolder.getMessageCount()
 131                                         : 0;
 132                         boolean hasSubFolders = (sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0
 133                                         ? sourceFolder.list().length != 0
 134                                         : false;
 135
 136                         Folder targetFolder;
 137                         if (targetSupportDualTypeFolders) {
 138                                 targetFolder = targetStore.getFolder(targetFolderFullName);
 139                                 if (!targetFolder.exists()) {
 140                                         targetFolder.create(Folder.HOLDS_FOLDERS | Folder.HOLDS_MESSAGES);
 141                                         logger.log(DEBUG, "Created HOLDS_FOLDERS | HOLDS_MESSAGES folder " + targetFolder.getFullName());
 142                                 }
 143
 144                         } else {
 145                                 if (hasSubFolders) {// has sub-folders
 146                                         if (messageCount == 0) {
 147                                                 targetFolder = targetStore.getFolder(targetFolderFullName);
 148                                                 if (!targetFolder.exists()) {
 149                                                         targetFolder.create(Folder.HOLDS_FOLDERS);
 150                                                         logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName());
 151                                                 }
 152                                         } else {// also has messages
 153                                                 Folder parentFolder = targetStore.getFolder(targetFolderFullName);
 154                                                 if (!parentFolder.exists()) {
 155                                                         parentFolder.create(Folder.HOLDS_FOLDERS);
 156                                                         logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName());
 157                                                 }
 158                                                 String miscFullName = targetFolderFullName + targetFolderSeparator + "_Misc";
 159                                                 targetFolder = targetStore.getFolder(miscFullName);
 160                                                 if (!targetFolder.exists()) {
 161                                                         targetFolder.create(Folder.HOLDS_MESSAGES);
 162                                                         logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName());
 163                                                 }
 164                                         }
 165                                 } else {// no sub-folders
 166                                         if (messageCount == 0) { // empty
 167                                                 logger.log(DEBUG, "Skip empty folder " + targetFolderFullName);
 168                                                 continue folders;
 169                                         }
 170                                         targetFolder = targetStore.getFolder(targetFolderFullName);
 171                                         if (!targetFolder.exists()) {
 172                                                 targetFolder.create(Folder.HOLDS_MESSAGES);
 173                                                 logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName());
 174                                         }
 175                                 }
 176                         }
 177
 178                         if (messageCount != 0) {
 179
 180                                 targetFolder.open(Folder.READ_WRITE);
 181                                 try {
 182                                         long begin = System.currentTimeMillis();
 183                                         sourceFolder.open(Folder.READ_ONLY);
 184                                         migrateFolder(sourceFolder, targetFolder);
 185                                         long duration = System.currentTimeMillis() - begin;
 186                                         logger.log(DEBUG, targetFolderFullName + " - Migration of " + messageCount + " messages took "
 187                                                         + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)");
 188                                 } finally {
 189                                         sourceFolder.close();
 190                                         targetFolder.close();
 191                                 }
 192                         }
 193
 194                         // recursive
 195                         if (hasSubFolders) {
 196                                 migrateFolders(sourceFolder, targetStore);
 197                         }
 198                 }
 199         }
 200
 201         protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 202                 folders: for (Folder folder : sourceFolder.list()) {
 203                         String folderName = folder.getName();
 204
 205                         if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
 206                                 // Make it configurable
 207                                 switch (folderName) {
 208                                 case "All Mail":
 209                                 case "Important":
 210                                         continue folders;
 211                                 default:
 212                                         // doe nothing
 213                                 }
 214                                 migrateFolderToFs(baseDir, folder);
 215                         }
 216                         if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
 217                                 migrateFoldersToFs(baseDir.resolve(folder.getName()), folder);
 218                         }
 219                 }
 220         }
 221
 222         protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 223
 224                 String folderName = sourceFolder.getName();
 225                 sourceFolder.open(Folder.READ_ONLY);
 226
 227                 Folder targetFolder = null;
 228                 try {
 229                         int messageCount = sourceFolder.getMessageCount();
 230                         logger.log(DEBUG, folderName + " - Message count : " + messageCount);
 231                         if (messageCount == 0)
 232                                 return;
 233 //                      logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
 234
 235                         boolean saveAsFiles = false;
 236
 237                         if (saveAsFiles) {
 238                                 Message messages[] = sourceFolder.getMessages();
 239
 240                                 for (int i = 0; i < messages.length; ++i) {
 241 //                                      logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
 242                                         Message msg = messages[i];
 243 //                                      String from = "unknown";
 244 //                                      if (msg.getReplyTo().length >= 1) {
 245 //                                              from = msg.getReplyTo()[0].toString();
 246 //                                      } else if (msg.getFrom().length >= 1) {
 247 //                                              from = msg.getFrom()[0].toString();
 248 //                                      }
 249                                         String subject = msg.getSubject();
 250                                         Instant sentDate = msg.getSentDate().toInstant();
 251 //                                      logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
 252                                         String fileName = sentDate + "  " + subject;
 253                                         Path file = baseDir.resolve(fileName);
 254                                         savePartsAsFiles(msg.getContent(), file);
 255                                 }
 256                         } else {
 257                                 long begin = System.currentTimeMillis();
 258                                 targetFolder = openMboxTargetFolder(sourceFolder, baseDir);
 259                                 migrateFolder(sourceFolder, targetFolder);
 260                                 long duration = System.currentTimeMillis() - begin;
 261                                 logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000)
 262                                                 + " s (" + (duration / messageCount) + " ms per message)");
 263                         }
 264                 } finally {
 265                         sourceFolder.close();
 266                         if (targetFolder != null)
 267                                 targetFolder.close();
 268                 }
 269         }
 270
 271         protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException {
 272                 String folderName = targetFolder.getName();
 273
 274                 int lastSourceNumber;
 275                 int currentTargetMessageCount = targetFolder.getMessageCount();
 276                 if (currentTargetMessageCount != 0) {
 277                         MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount);
 278                         logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage));
 279                         Date lastTargetSent = lastTargetMessage.getReceivedDate();
 280                         Message[] lastSourceMessage = sourceFolder
 281                                         .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID()));
 282                         if (lastSourceMessage.length == 0)
 283                                 throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID());
 284                         if (lastSourceMessage.length != 1) {
 285                                 for (Message msg : lastSourceMessage) {
 286                                         logger.log(ERROR, "Message " + describe(msg));
 287
 288                                 }
 289                                 throw new IllegalStateException(
 290                                                 lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant());
 291                         }
 292                         lastSourceNumber = lastSourceMessage[0].getMessageNumber();
 293                 } else {
 294                         lastSourceNumber = 0;
 295                 }
 296                 logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber);
 297
 298                 int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber;
 299
 300                 FetchProfile fetchProfile = new FetchProfile();
 301                 fetchProfile.add(FetchProfile.Item.FLAGS);
 302                 fetchProfile.add(FetchProfile.Item.ENVELOPE);
 303                 fetchProfile.add(FetchProfile.Item.CONTENT_INFO);
 304                 fetchProfile.add(FetchProfile.Item.SIZE);
 305                 if (sourceFolder instanceof IMAPFolder) {
 306                         // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
 307                         fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS);
 308                         fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE);
 309                 }
 310
 311                 int batchSize = 100;
 312                 int batchCount = countToRetrieve / batchSize;
 313                 if (countToRetrieve % batchSize != 0)
 314                         batchCount = batchCount + 1;
 315                 // int batchCount = 2; // for testing
 316                 for (int i = 0; i < batchCount; i++) {
 317                         long begin = System.currentTimeMillis();
 318
 319                         int start = lastSourceNumber + i * batchSize + 1;
 320                         int end = lastSourceNumber + (i + 1) * batchSize;
 321                         if (end >= (lastSourceNumber + countToRetrieve + 1))
 322                                 end = lastSourceNumber + countToRetrieve;
 323                         Message[] sourceMessages = sourceFolder.getMessages(start, end);
 324                         sourceFolder.fetch(sourceMessages, fetchProfile);
 325                         // targetFolder.appendMessages(sourceMessages);
 326                         // sourceFolder.copyMessages(sourceMessages,targetFolder);
 327
 328                         copyMessages(sourceMessages, targetFolder);
 329 //                      copyMessagesToMbox(sourceMessages, targetFolder);
 330
 331                         String describeLast = describe(sourceMessages[sourceMessages.length - 1]);
 332
 333 //              if (i % 10 == 9) {
 334                         // free memory from fetched messages
 335                         sourceFolder.close();
 336                         targetFolder.close();
 337
 338                         sourceFolder.open(Folder.READ_ONLY);
 339                         targetFolder.open(Folder.READ_WRITE);
 340 //                      logger.log(DEBUG, "Open/close folder in order to free memory");
 341 //              }
 342
 343                         long duration = System.currentTimeMillis() - begin;
 344                         logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, "
 345                                         + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast);
 346                 }
 347
 348                 return targetFolder;
 349         }
 350
 351         protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException {
 352                 String folderName = sourceFolder.getName();
 353                 if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE))
 354                         folderName = EmailUtils.INBOX;// Inbox
 355
 356                 Path targetDir = baseDir;// .resolve("mbox");
 357                 Files.createDirectories(targetDir);
 358                 Path targetPath;
 359                 if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) {
 360                         Path dir = targetDir.resolve(folderName);
 361                         Files.createDirectories(dir);
 362                         targetPath = dir.resolve("_Misc");
 363                 } else {
 364                         targetPath = targetDir.resolve(folderName);
 365                 }
 366                 if (!Files.exists(targetPath))
 367                         Files.createFile(targetPath);
 368                 URLName targetUrlName = new URLName("mbox:" + targetPath.toString());
 369                 Properties targetProperties = new Properties();
 370                 // targetProperties.setProperty("mail.mime.address.strict", "false");
 371                 Session targetSession = Session.getDefaultInstance(targetProperties);
 372                 Folder targetFolder = targetSession.getFolder(targetUrlName);
 373                 targetFolder.open(Folder.READ_WRITE);
 374
 375                 return targetFolder;
 376         }
 377
 378         protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException {
 379                 targetFolder.appendMessages(sourceMessages);
 380         }
 381
 382         protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder)
 383                         throws MessagingException, IOException {
 384                 Message[] targetMessages = new Message[sourceMessages.length];
 385                 for (int j = 0; j < sourceMessages.length; j++) {
 386                         MimeMessage sourceMm = (MimeMessage) sourceMessages[j];
 387                         InternetHeaders ih = new InternetHeaders();
 388                         for (Enumeration<String> e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) {
 389                                 ih.addHeaderLine(e.nextElement());
 390                         }
 391                         Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt");
 392                         Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt");
 393                         Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING);
 394
 395                         // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
 396                         // missing characters
 397                         try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1);
 398                                         BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) {
 399                                 int lineNumber = 0;
 400                                 String line = null;
 401                                 try {
 402                                         while ((line = reader.readLine()) != null) {
 403                                                 lineNumber++;
 404                                                 if (line.startsWith("From ")) {
 405                                                         writer.write(">" + line);
 406                                                         logger.log(DEBUG,
 407                                                                         "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line);
 408                                                 } else {
 409                                                         writer.write(line);
 410                                                 }
 411                                                 writer.newLine();
 412                                         }
 413                                 } catch (IOException e) {
 414                                         logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource);
 415                                         throw e;
 416                                 }
 417                         }
 418
 419                         MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih,
 420                                         new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(),
 421                                         EmailUtils.getUnixFrom(sourceMm), true);
 422                         targetMessages[j] = mboxMessage;
 423
 424                         // clean up
 425                         Files.delete(tmpFileSource);
 426                         Files.delete(tmpFileTarget);
 427                 }
 428                 targetFolder.appendMessages(targetMessages);
 429
 430         }
 431
 432         /** Save body parts and attachments as plain files. */
 433         protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException {
 434                 OutputStream out = null;
 435                 InputStream in = null;
 436                 try {
 437                         if (content instanceof Multipart) {
 438                                 Multipart multi = ((Multipart) content);
 439                                 int parts = multi.getCount();
 440                                 for (int j = 0; j < parts; ++j) {
 441                                         MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j);
 442                                         if (part.getContent() instanceof Multipart) {
 443                                                 // part-within-a-part, do some recursion...
 444                                                 savePartsAsFiles(part.getContent(), fileBase);
 445                                         } else {
 446                                                 String extension = "";
 447                                                 if (part.isMimeType("text/html")) {
 448                                                         extension = "html";
 449                                                 } else {
 450                                                         if (part.isMimeType("text/plain")) {
 451                                                                 extension = "txt";
 452                                                         } else {
 453                                                                 // Try to get the name of the attachment
 454                                                                 extension = part.getDataHandler().getName();
 455                                                         }
 456                                                 }
 457                                                 String filename = fileBase + "." + extension;
 458                                                 System.out.println("... " + filename);
 459                                                 out = new FileOutputStream(new File(filename));
 460                                                 in = part.getInputStream();
 461                                                 int k;
 462                                                 while ((k = in.read()) != -1) {
 463                                                         out.write(k);
 464                                                 }
 465                                         }
 466                                 }
 467                         }
 468                 } finally {
 469                         if (in != null) {
 470                                 in.close();
 471                         }
 472                         if (out != null) {
 473                                 out.flush();
 474                                 out.close();
 475                         }
 476                 }
 477         }
 478
 479         public void setSourceServer(String sourceServer) {
 480                 this.sourceServer = sourceServer;
 481         }
 482
 483         public void setSourceUsername(String sourceUsername) {
 484                 this.sourceUsername = sourceUsername;
 485         }
 486
 487         public void setSourcePassword(String sourcePassword) {
 488                 this.sourcePassword = sourcePassword;
 489         }
 490
 491         public void setTargetServer(String targetServer) {
 492                 this.targetServer = targetServer;
 493         }
 494
 495         public void setTargetUsername(String targetUsername) {
 496                 this.targetUsername = targetUsername;
 497         }
 498
 499         public void setTargetPassword(String targetPassword) {
 500                 this.targetPassword = targetPassword;
 501         }
 502
 503         public static void main(String args[]) throws Exception {
 504                 if (args.length < 6)
 505                         throw new IllegalArgumentException(
 506                                         "usage: <source IMAP server> <source username> <source password> <target IMAP server> <target username> <target password>");
 507                 String sourceServer = args[0];
 508                 String sourceUsername = args[1];
 509                 String sourcePassword = args[2];
 510                 String targetServer = args[3];
 511                 String targetUsername = args[4];
 512                 String targetPassword = args[5];
 513
 514                 EmailMigration emailMigration = new EmailMigration();
 515                 emailMigration.setSourceServer(sourceServer);
 516                 emailMigration.setSourceUsername(sourceUsername);
 517                 emailMigration.setSourcePassword(sourcePassword);
 518                 emailMigration.setTargetServer(targetServer);
 519                 emailMigration.setTargetUsername(targetUsername);
 520                 emailMigration.setTargetPassword(targetPassword);
 521
 522                 emailMigration.process();
 523         }
 524 }