org.argeo.slc.runtime/src/org/argeo/slc/mail/EmailMigration.java

   1 package org.argeo.slc.mail;
   2
   3 import static java.lang.System.Logger.Level.DEBUG;
   4 import static java.lang.System.Logger.Level.ERROR;
   5 import static org.argeo.slc.mail.EmailUtils.describe;
   6
   7 import java.io.BufferedReader;
   8 import java.io.BufferedWriter;
   9 import java.io.File;
  10 import java.io.FileOutputStream;
  11 import java.io.IOException;
  12 import java.io.InputStream;
  13 import java.io.OutputStream;
  14 import java.lang.System.Logger;
  15 import java.nio.charset.StandardCharsets;
  16 import java.nio.file.Files;
  17 import java.nio.file.Path;
  18 import java.nio.file.StandardCopyOption;
  19 import java.time.Instant;
  20 import java.util.Date;
  21 import java.util.Enumeration;
  22 import java.util.Properties;
  23
  24 import javax.mail.FetchProfile;
  25 import javax.mail.Folder;
  26 import javax.mail.Message;
  27 import javax.mail.MessagingException;
  28 import javax.mail.Multipart;
  29 import javax.mail.Session;
  30 import javax.mail.Store;
  31 import javax.mail.URLName;
  32 import javax.mail.internet.InternetHeaders;
  33 import javax.mail.internet.MimeBodyPart;
  34 import javax.mail.internet.MimeMessage;
  35 import javax.mail.search.HeaderTerm;
  36 import javax.mail.util.SharedFileInputStream;
  37
  38 import com.sun.mail.imap.IMAPFolder;
  39 import com.sun.mail.mbox.MboxFolder;
  40 import com.sun.mail.mbox.MboxMessage;
  41
  42 /** Migrates emails from one storage to the another one. */
  43 public class EmailMigration {
  44         private final static Logger logger = System.getLogger(EmailMigration.class.getName());
  45
  46 //      private String targetBaseDir;
  47
  48         private String sourceServer;
  49         private String sourceUsername;
  50         private String sourcePassword;
  51
  52         private String targetServer;
  53         private String targetUsername;
  54         private String targetPassword;
  55
  56         public void process() throws MessagingException, IOException {
  57 //              Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
  58
  59                 Store sourceStore = null;
  60                 try {
  61                         Properties sourceProperties = System.getProperties();
  62                         sourceProperties.setProperty("mail.store.protocol", "imaps");
  63
  64                         Session sourceSession = Session.getInstance(sourceProperties, null);
  65                         // session.setDebug(true);
  66                         sourceStore = sourceSession.getStore("imaps");
  67                         sourceStore.connect(sourceServer, sourceUsername, sourcePassword);
  68
  69                         Folder defaultFolder = sourceStore.getDefaultFolder();
  70 //                      migrateFolders(baseDir, defaultFolder);
  71
  72                         // Always start with Inbox
  73 //                      Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
  74 //                      migrateFolder(baseDir, inboxFolder);
  75
  76                         Properties targetProperties = System.getProperties();
  77                         targetProperties.setProperty("mail.imap.starttls.enable", "true");
  78                         targetProperties.setProperty("mail.imap.auth", "true");
  79
  80                         Session targetSession = Session.getInstance(targetProperties, null);
  81                         // session.setDebug(true);
  82                         Store targetStore = targetSession.getStore("imap");
  83                         targetStore.connect(targetServer, targetUsername, targetPassword);
  84
  85 //                      Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX);
  86 //                      logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount());
  87 //                      logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount());
  88
  89                         migrateFolders(defaultFolder, targetStore);
  90                 } finally {
  91                         if (sourceStore != null)
  92                                 sourceStore.close();
  93
  94                 }
  95         }
  96
  97         protected void migrateFolders(Folder sourceFolder, Store targetStore) throws MessagingException, IOException {
  98                 folders: for (Folder folder : sourceFolder.list()) {
  99                         String folderName = folder.getName();
 100
 101                         String folderFullName = folder.getFullName();
 102
 103                         // GMail specific
 104                         if (folderFullName.equals("[Gmail]")) {
 105                                 migrateFolders(folder, targetStore);
 106                                 continue folders;
 107                         }
 108                         if (folderFullName.startsWith("[Gmail]")) {
 109                                 // Make it configurable
 110                                 switch (folderName) {
 111                                 case "All Mail":
 112                                 case "Important":
 113                                 case "Spam":
 114                                         continue folders;
 115                                 default:
 116                                         // does nothing
 117                                 }
 118                                 folderFullName = folder.getName();
 119                         }
 120
 121                         int messageCount = (folder.getType() & Folder.HOLDS_MESSAGES) != 0 ? folder.getMessageCount() : 0;
 122                         boolean hasSubFolders = (folder.getType() & Folder.HOLDS_FOLDERS) != 0 ? folder.list().length != 0 : false;
 123                         Folder targetFolder;
 124                         if (hasSubFolders) {// has sub-folders
 125                                 if (messageCount == 0) {
 126                                         targetFolder = targetStore.getFolder(folderFullName);
 127                                         if (!targetFolder.exists()) {
 128                                                 targetFolder.create(Folder.HOLDS_FOLDERS);
 129                                                 logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + targetFolder.getFullName());
 130                                         }
 131                                 } else {// also has messages
 132                                         Folder parentFolder = targetStore.getFolder(folderFullName);
 133                                         if (!parentFolder.exists()) {
 134                                                 parentFolder.create(Folder.HOLDS_FOLDERS);
 135                                                 logger.log(DEBUG, "Created HOLDS_FOLDERS folder " + parentFolder.getFullName());
 136                                         }
 137                                         String miscFullName = folderFullName + "/_Misc";
 138                                         targetFolder = targetStore.getFolder(miscFullName);
 139                                         if (!targetFolder.exists()) {
 140                                                 targetFolder.create(Folder.HOLDS_MESSAGES);
 141                                                 logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName());
 142                                         }
 143                                 }
 144                         } else {// no sub-folders
 145                                 if (messageCount == 0) { // empty
 146                                         logger.log(DEBUG, "Skip empty folder " + folderFullName);
 147                                         continue folders;
 148                                 }
 149                                 targetFolder = targetStore.getFolder(folderFullName);
 150                                 if (!targetFolder.exists()) {
 151                                         targetFolder.create(Folder.HOLDS_MESSAGES);
 152                                         logger.log(DEBUG, "Created HOLDS_MESSAGES folder " + targetFolder.getFullName());
 153                                 }
 154                         }
 155
 156                         if (messageCount != 0) {
 157
 158                                 targetFolder.open(Folder.READ_WRITE);
 159                                 try {
 160                                         long begin = System.currentTimeMillis();
 161                                         folder.open(Folder.READ_ONLY);
 162                                         migrateFolder(folder, targetFolder);
 163                                         long duration = System.currentTimeMillis() - begin;
 164                                         logger.log(DEBUG, folderFullName + " - Migration of " + messageCount + " messages took "
 165                                                         + (duration / 1000) + " s (" + (duration / messageCount) + " ms per message)");
 166                                 } finally {
 167                                         folder.close();
 168                                         targetFolder.close();
 169                                 }
 170                         }
 171
 172                         // recursive
 173                         if (hasSubFolders) {
 174                                 migrateFolders(folder, targetStore);
 175                         }
 176                 }
 177         }
 178
 179         protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 180                 folders: for (Folder folder : sourceFolder.list()) {
 181                         String folderName = folder.getName();
 182
 183                         if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
 184                                 // Make it configurable
 185                                 switch (folderName) {
 186                                 case "All Mail":
 187                                 case "Important":
 188                                         continue folders;
 189                                 default:
 190                                         // doe nothing
 191                                 }
 192                                 migrateFolderToFs(baseDir, folder);
 193                         }
 194                         if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
 195                                 migrateFoldersToFs(baseDir.resolve(folder.getName()), folder);
 196                         }
 197                 }
 198         }
 199
 200         protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
 201
 202                 String folderName = sourceFolder.getName();
 203                 sourceFolder.open(Folder.READ_ONLY);
 204
 205                 Folder targetFolder = null;
 206                 try {
 207                         int messageCount = sourceFolder.getMessageCount();
 208                         logger.log(DEBUG, folderName + " - Message count : " + messageCount);
 209                         if (messageCount == 0)
 210                                 return;
 211 //                      logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
 212
 213                         boolean saveAsFiles = false;
 214
 215                         if (saveAsFiles) {
 216                                 Message messages[] = sourceFolder.getMessages();
 217
 218                                 for (int i = 0; i < messages.length; ++i) {
 219 //                                      logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
 220                                         Message msg = messages[i];
 221 //                                      String from = "unknown";
 222 //                                      if (msg.getReplyTo().length >= 1) {
 223 //                                              from = msg.getReplyTo()[0].toString();
 224 //                                      } else if (msg.getFrom().length >= 1) {
 225 //                                              from = msg.getFrom()[0].toString();
 226 //                                      }
 227                                         String subject = msg.getSubject();
 228                                         Instant sentDate = msg.getSentDate().toInstant();
 229 //                                      logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
 230                                         String fileName = sentDate + "  " + subject;
 231                                         Path file = baseDir.resolve(fileName);
 232                                         savePartsAsFiles(msg.getContent(), file);
 233                                 }
 234                         }
 235                         else {
 236                                 long begin = System.currentTimeMillis();
 237                                 targetFolder = openMboxTargetFolder(sourceFolder, baseDir);
 238                                 migrateFolder(sourceFolder, targetFolder);
 239                                 long duration = System.currentTimeMillis() - begin;
 240                                 logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000)
 241                                                 + " s (" + (duration / messageCount) + " ms per message)");
 242                         }
 243                 } finally {
 244                         sourceFolder.close();
 245                         if (targetFolder != null)
 246                                 targetFolder.close();
 247                 }
 248         }
 249
 250         protected Folder migrateFolder(Folder sourceFolder, Folder targetFolder) throws MessagingException, IOException {
 251                 String folderName = targetFolder.getName();
 252
 253                 int lastSourceNumber;
 254                 int currentTargetMessageCount = targetFolder.getMessageCount();
 255                 if (currentTargetMessageCount != 0) {
 256                         MimeMessage lastTargetMessage = (MimeMessage) targetFolder.getMessage(currentTargetMessageCount);
 257                         logger.log(DEBUG, folderName + " - Last target message " + describe(lastTargetMessage));
 258                         Date lastTargetSent = lastTargetMessage.getReceivedDate();
 259                         Message[] lastSourceMessage = sourceFolder
 260                                         .search(new HeaderTerm(EmailUtils.MESSAGE_ID, lastTargetMessage.getMessageID()));
 261                         if (lastSourceMessage.length == 0)
 262                                 throw new IllegalStateException("No message found with message ID " + lastTargetMessage.getMessageID());
 263                         if (lastSourceMessage.length != 1) {
 264                                 for (Message msg : lastSourceMessage) {
 265                                         logger.log(ERROR, "Message " + describe(msg));
 266
 267                                 }
 268                                 throw new IllegalStateException(
 269                                                 lastSourceMessage.length + " messages found with received date " + lastTargetSent.toInstant());
 270                         }
 271                         lastSourceNumber = lastSourceMessage[0].getMessageNumber();
 272                 } else {
 273                         lastSourceNumber = 0;
 274                 }
 275                 logger.log(DEBUG, folderName + " - Last source message number " + lastSourceNumber);
 276
 277                 int countToRetrieve = sourceFolder.getMessageCount() - lastSourceNumber;
 278
 279                 FetchProfile fetchProfile = new FetchProfile();
 280                 fetchProfile.add(FetchProfile.Item.FLAGS);
 281                 fetchProfile.add(FetchProfile.Item.ENVELOPE);
 282                 fetchProfile.add(FetchProfile.Item.CONTENT_INFO);
 283                 fetchProfile.add(FetchProfile.Item.SIZE);
 284                 if (sourceFolder instanceof IMAPFolder) {
 285                         // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
 286                         fetchProfile.add(IMAPFolder.FetchProfileItem.HEADERS);
 287                         fetchProfile.add(IMAPFolder.FetchProfileItem.MESSAGE);
 288                 }
 289
 290                 int batchSize = 100;
 291                 int batchCount = countToRetrieve / batchSize;
 292                 if (countToRetrieve % batchSize != 0)
 293                         batchCount = batchCount + 1;
 294                 // int batchCount = 2; // for testing
 295                 for (int i = 0; i < batchCount; i++) {
 296                         long begin = System.currentTimeMillis();
 297
 298                         int start = lastSourceNumber + i * batchSize + 1;
 299                         int end = lastSourceNumber + (i + 1) * batchSize;
 300                         if (end >= (lastSourceNumber + countToRetrieve + 1))
 301                                 end = lastSourceNumber + countToRetrieve;
 302                         Message[] sourceMessages = sourceFolder.getMessages(start, end);
 303                         sourceFolder.fetch(sourceMessages, fetchProfile);
 304                         // targetFolder.appendMessages(sourceMessages);
 305                         // sourceFolder.copyMessages(sourceMessages,targetFolder);
 306
 307                         copyMessages(sourceMessages, targetFolder);
 308 //                      copyMessagesToMbox(sourceMessages, targetFolder);
 309
 310                         String describeLast = describe(sourceMessages[sourceMessages.length - 1]);
 311
 312 //              if (i % 10 == 9) {
 313                         // free memory from fetched messages
 314                         sourceFolder.close();
 315                         targetFolder.close();
 316
 317                         sourceFolder.open(Folder.READ_ONLY);
 318                         targetFolder.open(Folder.READ_WRITE);
 319 //                      logger.log(DEBUG, "Open/close folder in order to free memory");
 320 //              }
 321
 322                         long duration = System.currentTimeMillis() - begin;
 323                         logger.log(DEBUG, folderName + " - batch " + i + " took " + (duration / 1000) + " s, "
 324                                         + (duration / (end - start + 1)) + " ms per message. Last message " + describeLast);
 325                 }
 326
 327                 return targetFolder;
 328         }
 329
 330         protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException {
 331                 String folderName = sourceFolder.getName();
 332                 if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE))
 333                         folderName = EmailUtils.INBOX;// Inbox
 334
 335                 Path targetDir = baseDir;// .resolve("mbox");
 336                 Files.createDirectories(targetDir);
 337                 Path targetPath;
 338                 if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) {
 339                         Path dir = targetDir.resolve(folderName);
 340                         Files.createDirectories(dir);
 341                         targetPath = dir.resolve("_Misc");
 342                 } else {
 343                         targetPath = targetDir.resolve(folderName);
 344                 }
 345                 if (!Files.exists(targetPath))
 346                         Files.createFile(targetPath);
 347                 URLName targetUrlName = new URLName("mbox:" + targetPath.toString());
 348                 Properties targetProperties = new Properties();
 349                 // targetProperties.setProperty("mail.mime.address.strict", "false");
 350                 Session targetSession = Session.getDefaultInstance(targetProperties);
 351                 Folder targetFolder = targetSession.getFolder(targetUrlName);
 352                 targetFolder.open(Folder.READ_WRITE);
 353
 354                 return targetFolder;
 355         }
 356
 357         protected void copyMessages(Message[] sourceMessages, Folder targetFolder) throws MessagingException {
 358                 targetFolder.appendMessages(sourceMessages);
 359         }
 360
 361         protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder)
 362                         throws MessagingException, IOException {
 363                 Message[] targetMessages = new Message[sourceMessages.length];
 364                 for (int j = 0; j < sourceMessages.length; j++) {
 365                         MimeMessage sourceMm = (MimeMessage) sourceMessages[j];
 366                         InternetHeaders ih = new InternetHeaders();
 367                         for (Enumeration<String> e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) {
 368                                 ih.addHeaderLine(e.nextElement());
 369                         }
 370                         Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt");
 371                         Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt");
 372                         Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING);
 373
 374                         // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
 375                         // missing characters
 376                         try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1);
 377                                         BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) {
 378                                 int lineNumber = 0;
 379                                 String line = null;
 380                                 try {
 381                                         while ((line = reader.readLine()) != null) {
 382                                                 lineNumber++;
 383                                                 if (line.startsWith("From ")) {
 384                                                         writer.write(">" + line);
 385                                                         logger.log(DEBUG,
 386                                                                         "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line);
 387                                                 } else {
 388                                                         writer.write(line);
 389                                                 }
 390                                                 writer.newLine();
 391                                         }
 392                                 } catch (IOException e) {
 393                                         logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource);
 394                                         throw e;
 395                                 }
 396                         }
 397
 398                         MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih,
 399                                         new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(),
 400                                         EmailUtils.getUnixFrom(sourceMm), true);
 401                         targetMessages[j] = mboxMessage;
 402
 403                         // clean up
 404                         Files.delete(tmpFileSource);
 405                         Files.delete(tmpFileTarget);
 406                 }
 407                 targetFolder.appendMessages(targetMessages);
 408
 409         }
 410
 411         /** Save body parts and attachments as plain files. */
 412         protected void savePartsAsFiles(Object content, Path fileBase) throws IOException, MessagingException {
 413                 OutputStream out = null;
 414                 InputStream in = null;
 415                 try {
 416                         if (content instanceof Multipart) {
 417                                 Multipart multi = ((Multipart) content);
 418                                 int parts = multi.getCount();
 419                                 for (int j = 0; j < parts; ++j) {
 420                                         MimeBodyPart part = (MimeBodyPart) multi.getBodyPart(j);
 421                                         if (part.getContent() instanceof Multipart) {
 422                                                 // part-within-a-part, do some recursion...
 423                                                 savePartsAsFiles(part.getContent(), fileBase);
 424                                         } else {
 425                                                 String extension = "";
 426                                                 if (part.isMimeType("text/html")) {
 427                                                         extension = "html";
 428                                                 } else {
 429                                                         if (part.isMimeType("text/plain")) {
 430                                                                 extension = "txt";
 431                                                         } else {
 432                                                                 // Try to get the name of the attachment
 433                                                                 extension = part.getDataHandler().getName();
 434                                                         }
 435                                                 }
 436                                                 String filename = fileBase + "." + extension;
 437                                                 System.out.println("... " + filename);
 438                                                 out = new FileOutputStream(new File(filename));
 439                                                 in = part.getInputStream();
 440                                                 int k;
 441                                                 while ((k = in.read()) != -1) {
 442                                                         out.write(k);
 443                                                 }
 444                                         }
 445                                 }
 446                         }
 447                 } finally {
 448                         if (in != null) {
 449                                 in.close();
 450                         }
 451                         if (out != null) {
 452                                 out.flush();
 453                                 out.close();
 454                         }
 455                 }
 456         }
 457
 458         public void setSourceServer(String sourceServer) {
 459                 this.sourceServer = sourceServer;
 460         }
 461
 462         public void setSourceUsername(String sourceUsername) {
 463                 this.sourceUsername = sourceUsername;
 464         }
 465
 466         public void setSourcePassword(String sourcePassword) {
 467                 this.sourcePassword = sourcePassword;
 468         }
 469
 470         public void setTargetServer(String targetServer) {
 471                 this.targetServer = targetServer;
 472         }
 473
 474         public void setTargetUsername(String targetUsername) {
 475                 this.targetUsername = targetUsername;
 476         }
 477
 478         public void setTargetPassword(String targetPassword) {
 479                 this.targetPassword = targetPassword;
 480         }
 481
 482         public static void main(String args[]) throws Exception {
 483                 if (args.length < 6)
 484                         throw new IllegalArgumentException(
 485                                         "usage: <source IMAP server> <source username> <source password> <target IMAP server> <target username> <target password>");
 486                 String sourceServer = args[0];
 487                 String sourceUsername = args[1];
 488                 String sourcePassword = args[2];
 489                 String targetServer = args[3];
 490                 String targetUsername = args[4];
 491                 String targetPassword = args[5];
 492
 493                 EmailMigration emailMigration = new EmailMigration();
 494                 emailMigration.setSourceServer(sourceServer);
 495                 emailMigration.setSourceUsername(sourceUsername);
 496                 emailMigration.setSourcePassword(sourcePassword);
 497                 emailMigration.setTargetServer(targetServer);
 498                 emailMigration.setTargetUsername(targetUsername);
 499                 emailMigration.setTargetPassword(targetPassword);
 500
 501                 emailMigration.process();
 502         }
 503 }