1 package org
.argeo
.slc
.mail
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static java
.lang
.System
.Logger
.Level
.ERROR
;
5 import static org
.argeo
.slc
.mail
.EmailUtils
.describe
;
8 import java
.io
.FileOutputStream
;
9 import java
.io
.IOException
;
10 import java
.io
.InputStream
;
11 import java
.io
.OutputStream
;
12 import java
.lang
.System
.Logger
;
13 import java
.nio
.file
.Path
;
14 import java
.util
.Date
;
15 import java
.util
.Properties
;
17 import javax
.mail
.FetchProfile
;
18 import javax
.mail
.Folder
;
19 import javax
.mail
.Message
;
20 import javax
.mail
.MessagingException
;
21 import javax
.mail
.Multipart
;
22 import javax
.mail
.Session
;
23 import javax
.mail
.Store
;
24 import javax
.mail
.internet
.MimeBodyPart
;
25 import javax
.mail
.internet
.MimeMessage
;
26 import javax
.mail
.search
.HeaderTerm
;
28 import com
.sun
.mail
.imap
.IMAPFolder
;
30 /** Migrates emails from one storage to the another one. */
31 public class EmailMigration
{
32 private final static Logger logger
= System
.getLogger(EmailMigration
.class.getName());
34 // private String targetBaseDir;
36 private String sourceServer
;
37 private String sourceUsername
;
38 private String sourcePassword
;
40 private String targetServer
;
41 private String targetUsername
;
42 private String targetPassword
;
44 public void process() throws MessagingException
, IOException
{
45 // Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
47 Store sourceStore
= null;
49 Properties sourceProperties
= System
.getProperties();
50 sourceProperties
.setProperty("mail.store.protocol", "imaps");
52 Session sourceSession
= Session
.getInstance(sourceProperties
, null);
53 // session.setDebug(true);
54 sourceStore
= sourceSession
.getStore("imaps");
55 sourceStore
.connect(sourceServer
, sourceUsername
, sourcePassword
);
57 Folder defaultFolder
= sourceStore
.getDefaultFolder();
58 // migrateFolders(baseDir, defaultFolder);
60 // Always start with Inbox
61 // Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
62 // migrateFolder(baseDir, inboxFolder);
64 Properties targetProperties
= System
.getProperties();
65 targetProperties
.setProperty("mail.imap.starttls.enable", "true");
66 targetProperties
.setProperty("mail.imap.auth", "true");
68 Session targetSession
= Session
.getInstance(targetProperties
, null);
69 // session.setDebug(true);
70 Store targetStore
= targetSession
.getStore("imap");
71 targetStore
.connect(targetServer
, targetUsername
, targetPassword
);
73 // Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX);
74 // logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount());
75 // logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount());
77 migrateFolders(defaultFolder
, targetStore
);
79 if (sourceStore
!= null)
85 protected void migrateFolders(Folder sourceFolder
, Store targetStore
) throws MessagingException
, IOException
{
86 folders
: for (Folder folder
: sourceFolder
.list()) {
87 String folderName
= folder
.getName();
89 String folderFullName
= folder
.getFullName();
92 if (folderFullName
.equals("[Gmail]")) {
93 migrateFolders(folder
, targetStore
);
96 if (folderFullName
.startsWith("[Gmail]")) {
97 // Make it configurable
106 folderFullName
= folder
.getName();
109 int messageCount
= (folder
.getType() & Folder
.HOLDS_MESSAGES
) != 0 ? folder
.getMessageCount() : 0;
110 boolean hasSubFolders
= (folder
.getType() & Folder
.HOLDS_FOLDERS
) != 0 ? folder
.list().length
!= 0 : false;
112 if (hasSubFolders
) {// has sub-folders
113 if (messageCount
== 0) {
114 targetFolder
= targetStore
.getFolder(folderFullName
);
115 if (!targetFolder
.exists()) {
116 targetFolder
.create(Folder
.HOLDS_FOLDERS
);
117 logger
.log(DEBUG
, "Created HOLDS_FOLDERS folder " + targetFolder
.getFullName());
119 } else {// also has messages
120 Folder parentFolder
= targetStore
.getFolder(folderFullName
);
121 if (!parentFolder
.exists()) {
122 parentFolder
.create(Folder
.HOLDS_FOLDERS
);
123 logger
.log(DEBUG
, "Created HOLDS_FOLDERS folder " + parentFolder
.getFullName());
125 String miscFullName
= folderFullName
+ "/_Misc";
126 targetFolder
= targetStore
.getFolder(miscFullName
);
127 if (!targetFolder
.exists()) {
128 targetFolder
.create(Folder
.HOLDS_MESSAGES
);
129 logger
.log(DEBUG
, "Created HOLDS_MESSAGES folder " + targetFolder
.getFullName());
132 } else {// no sub-folders
133 if (messageCount
== 0) { // empty
134 logger
.log(DEBUG
, "Skip empty folder " + folderFullName
);
137 targetFolder
= targetStore
.getFolder(folderFullName
);
138 if (!targetFolder
.exists()) {
139 targetFolder
.create(Folder
.HOLDS_MESSAGES
);
140 logger
.log(DEBUG
, "Created HOLDS_MESSAGES folder " + targetFolder
.getFullName());
144 if (messageCount
!= 0) {
146 targetFolder
.open(Folder
.READ_WRITE
);
148 long begin
= System
.currentTimeMillis();
149 folder
.open(Folder
.READ_ONLY
);
150 migrateFolder(folder
, targetFolder
);
151 long duration
= System
.currentTimeMillis() - begin
;
152 logger
.log(DEBUG
, folderFullName
+ " - Migration of " + messageCount
+ " messages took "
153 + (duration
/ 1000) + " s (" + (duration
/ messageCount
) + " ms per message)");
156 targetFolder
.close();
162 migrateFolders(folder
, targetStore
);
167 // protected void migrateFoldersToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
168 // folders: for (Folder folder : sourceFolder.list()) {
169 // String folderName = folder.getName();
171 // if ((folder.getType() & Folder.HOLDS_MESSAGES) != 0) {
172 // // Make it configurable
173 // switch (folderName) {
180 // migrateFolderToFs(baseDir, folder);
182 // if ((folder.getType() & Folder.HOLDS_FOLDERS) != 0) {
183 // migrateFoldersToFs(baseDir.resolve(folder.getName()), folder);
188 // protected void migrateFolderToFs(Path baseDir, Folder sourceFolder) throws MessagingException, IOException {
190 // String folderName = sourceFolder.getName();
191 // sourceFolder.open(Folder.READ_ONLY);
193 // Folder targetFolder = null;
195 // int messageCount = sourceFolder.getMessageCount();
196 // logger.log(DEBUG, folderName + " - Message count : " + messageCount);
197 // if (messageCount == 0)
199 //// logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
201 // boolean saveAsFiles = false;
203 // if (saveAsFiles) {
204 // Message messages[] = sourceFolder.getMessages();
206 // for (int i = 0; i < messages.length; ++i) {
207 //// logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
208 // Message msg = messages[i];
209 //// String from = "unknown";
210 //// if (msg.getReplyTo().length >= 1) {
211 //// from = msg.getReplyTo()[0].toString();
212 //// } else if (msg.getFrom().length >= 1) {
213 //// from = msg.getFrom()[0].toString();
215 // String subject = msg.getSubject();
216 // Instant sentDate = msg.getSentDate().toInstant();
217 //// logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
218 // String fileName = sentDate + " " + subject;
219 // Path file = baseDir.resolve(fileName);
220 // savePartsAsFiles(msg.getContent(), file);
224 // long begin = System.currentTimeMillis();
225 // targetFolder = openMboxTargetFolder(sourceFolder, baseDir);
226 // migrateFolder(sourceFolder, targetFolder);
227 // long duration = System.currentTimeMillis() - begin;
228 // logger.log(DEBUG, folderName + " - Migration of " + messageCount + " messages took " + (duration / 1000)
229 // + " s (" + (duration / messageCount) + " ms per message)");
232 // sourceFolder.close();
233 // if (targetFolder != null)
234 // targetFolder.close();
238 protected Folder
migrateFolder(Folder sourceFolder
, Folder targetFolder
) throws MessagingException
, IOException
{
239 String folderName
= targetFolder
.getName();
241 int lastSourceNumber
;
242 int currentTargetMessageCount
= targetFolder
.getMessageCount();
243 if (currentTargetMessageCount
!= 0) {
244 MimeMessage lastTargetMessage
= (MimeMessage
) targetFolder
.getMessage(currentTargetMessageCount
);
245 logger
.log(DEBUG
, folderName
+ " - Last target message " + describe(lastTargetMessage
));
246 Date lastTargetSent
= lastTargetMessage
.getReceivedDate();
247 Message
[] lastSourceMessage
= sourceFolder
248 .search(new HeaderTerm(EmailUtils
.MESSAGE_ID
, lastTargetMessage
.getMessageID()));
249 if (lastSourceMessage
.length
== 0)
250 throw new IllegalStateException("No message found with message ID " + lastTargetMessage
.getMessageID());
251 if (lastSourceMessage
.length
!= 1) {
252 for (Message msg
: lastSourceMessage
) {
253 logger
.log(ERROR
, "Message " + describe(msg
));
256 throw new IllegalStateException(
257 lastSourceMessage
.length
+ " messages found with received date " + lastTargetSent
.toInstant());
259 lastSourceNumber
= lastSourceMessage
[0].getMessageNumber();
261 lastSourceNumber
= 0;
263 logger
.log(DEBUG
, folderName
+ " - Last source message number " + lastSourceNumber
);
265 int countToRetrieve
= sourceFolder
.getMessageCount() - lastSourceNumber
;
267 FetchProfile fetchProfile
= new FetchProfile();
268 fetchProfile
.add(FetchProfile
.Item
.FLAGS
);
269 fetchProfile
.add(FetchProfile
.Item
.ENVELOPE
);
270 fetchProfile
.add(FetchProfile
.Item
.CONTENT_INFO
);
271 fetchProfile
.add(FetchProfile
.Item
.SIZE
);
272 if (sourceFolder
instanceof IMAPFolder
) {
273 // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
274 fetchProfile
.add(IMAPFolder
.FetchProfileItem
.HEADERS
);
275 fetchProfile
.add(IMAPFolder
.FetchProfileItem
.MESSAGE
);
279 int batchCount
= countToRetrieve
/ batchSize
;
280 if (countToRetrieve
% batchSize
!= 0)
281 batchCount
= batchCount
+ 1;
282 // int batchCount = 2; // for testing
283 for (int i
= 0; i
< batchCount
; i
++) {
284 long begin
= System
.currentTimeMillis();
286 int start
= lastSourceNumber
+ i
* batchSize
+ 1;
287 int end
= lastSourceNumber
+ (i
+ 1) * batchSize
;
288 if (end
>= (lastSourceNumber
+ countToRetrieve
+ 1))
289 end
= lastSourceNumber
+ countToRetrieve
;
290 Message
[] sourceMessages
= sourceFolder
.getMessages(start
, end
);
291 sourceFolder
.fetch(sourceMessages
, fetchProfile
);
292 // targetFolder.appendMessages(sourceMessages);
293 // sourceFolder.copyMessages(sourceMessages,targetFolder);
295 copyMessages(sourceMessages
, targetFolder
);
296 // copyMessagesToMbox(sourceMessages, targetFolder);
298 String describeLast
= describe(sourceMessages
[sourceMessages
.length
- 1]);
300 // if (i % 10 == 9) {
301 // free memory from fetched messages
302 sourceFolder
.close();
303 targetFolder
.close();
305 sourceFolder
.open(Folder
.READ_ONLY
);
306 targetFolder
.open(Folder
.READ_WRITE
);
307 // logger.log(DEBUG, "Open/close folder in order to free memory");
310 long duration
= System
.currentTimeMillis() - begin
;
311 logger
.log(DEBUG
, folderName
+ " - batch " + i
+ " took " + (duration
/ 1000) + " s, "
312 + (duration
/ (end
- start
+ 1)) + " ms per message. Last message " + describeLast
);
318 // protected Folder openMboxTargetFolder(Folder sourceFolder, Path baseDir) throws MessagingException, IOException {
319 // String folderName = sourceFolder.getName();
320 // if (sourceFolder.getName().equals(EmailUtils.INBOX_UPPER_CASE))
321 // folderName = EmailUtils.INBOX;// Inbox
323 // Path targetDir = baseDir;// .resolve("mbox");
324 // Files.createDirectories(targetDir);
326 // if (((sourceFolder.getType() & Folder.HOLDS_FOLDERS) != 0) && sourceFolder.list().length != 0) {
327 // Path dir = targetDir.resolve(folderName);
328 // Files.createDirectories(dir);
329 // targetPath = dir.resolve("_Misc");
331 // targetPath = targetDir.resolve(folderName);
333 // if (!Files.exists(targetPath))
334 // Files.createFile(targetPath);
335 // URLName targetUrlName = new URLName("mbox:" + targetPath.toString());
336 // Properties targetProperties = new Properties();
337 // // targetProperties.setProperty("mail.mime.address.strict", "false");
338 // Session targetSession = Session.getDefaultInstance(targetProperties);
339 // Folder targetFolder = targetSession.getFolder(targetUrlName);
340 // targetFolder.open(Folder.READ_WRITE);
342 // return targetFolder;
345 protected void copyMessages(Message
[] sourceMessages
, Folder targetFolder
) throws MessagingException
{
346 targetFolder
.appendMessages(sourceMessages
);
349 // protected void copyMessagesToMbox(Message[] sourceMessages, Folder targetFolder)
350 // throws MessagingException, IOException {
351 // Message[] targetMessages = new Message[sourceMessages.length];
352 // for (int j = 0; j < sourceMessages.length; j++) {
353 // MimeMessage sourceMm = (MimeMessage) sourceMessages[j];
354 // InternetHeaders ih = new InternetHeaders();
355 // for (Enumeration<String> e = sourceMm.getAllHeaderLines(); e.hasMoreElements();) {
356 // ih.addHeaderLine(e.nextElement());
358 // Path tmpFileSource = Files.createTempFile("argeo-mbox-source", ".txt");
359 // Path tmpFileTarget = Files.createTempFile("argeo-mbox-target", ".txt");
360 // Files.copy(sourceMm.getRawInputStream(), tmpFileSource, StandardCopyOption.REPLACE_EXISTING);
362 // // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
363 // // missing characters
364 // try (BufferedReader reader = Files.newBufferedReader(tmpFileSource, StandardCharsets.ISO_8859_1);
365 // BufferedWriter writer = Files.newBufferedWriter(tmpFileTarget, StandardCharsets.ISO_8859_1);) {
366 // int lineNumber = 0;
367 // String line = null;
369 // while ((line = reader.readLine()) != null) {
371 // if (line.startsWith("From ")) {
372 // writer.write(">" + line);
374 // "Fix line " + lineNumber + " in " + EmailUtils.describe(sourceMm) + ": " + line);
376 // writer.write(line);
380 // } catch (IOException e) {
381 // logger.log(ERROR, "Error around line " + lineNumber + " of " + tmpFileSource);
386 // MboxMessage mboxMessage = new MboxMessage((MboxFolder) targetFolder, ih,
387 // new SharedFileInputStream(tmpFileTarget.toFile()), sourceMm.getMessageNumber(),
388 // EmailUtils.getUnixFrom(sourceMm), true);
389 // targetMessages[j] = mboxMessage;
392 // Files.delete(tmpFileSource);
393 // Files.delete(tmpFileTarget);
395 // targetFolder.appendMessages(targetMessages);
399 /** Save body parts and attachments as plain files. */
400 protected void savePartsAsFiles(Object content
, Path fileBase
) throws IOException
, MessagingException
{
401 OutputStream out
= null;
402 InputStream in
= null;
404 if (content
instanceof Multipart
) {
405 Multipart multi
= ((Multipart
) content
);
406 int parts
= multi
.getCount();
407 for (int j
= 0; j
< parts
; ++j
) {
408 MimeBodyPart part
= (MimeBodyPart
) multi
.getBodyPart(j
);
409 if (part
.getContent() instanceof Multipart
) {
410 // part-within-a-part, do some recursion...
411 savePartsAsFiles(part
.getContent(), fileBase
);
413 String extension
= "";
414 if (part
.isMimeType("text/html")) {
417 if (part
.isMimeType("text/plain")) {
420 // Try to get the name of the attachment
421 extension
= part
.getDataHandler().getName();
424 String filename
= fileBase
+ "." + extension
;
425 System
.out
.println("... " + filename
);
426 out
= new FileOutputStream(new File(filename
));
427 in
= part
.getInputStream();
429 while ((k
= in
.read()) != -1) {
446 public void setSourceServer(String sourceServer
) {
447 this.sourceServer
= sourceServer
;
450 public void setSourceUsername(String sourceUsername
) {
451 this.sourceUsername
= sourceUsername
;
454 public void setSourcePassword(String sourcePassword
) {
455 this.sourcePassword
= sourcePassword
;
458 public void setTargetServer(String targetServer
) {
459 this.targetServer
= targetServer
;
462 public void setTargetUsername(String targetUsername
) {
463 this.targetUsername
= targetUsername
;
466 public void setTargetPassword(String targetPassword
) {
467 this.targetPassword
= targetPassword
;
470 public static void main(String args
[]) throws Exception
{
472 throw new IllegalArgumentException(
473 "usage: <source IMAP server> <source username> <source password> <target IMAP server> <target username> <target password>");
474 String sourceServer
= args
[0];
475 String sourceUsername
= args
[1];
476 String sourcePassword
= args
[2];
477 String targetServer
= args
[3];
478 String targetUsername
= args
[4];
479 String targetPassword
= args
[5];
481 EmailMigration emailMigration
= new EmailMigration();
482 emailMigration
.setSourceServer(sourceServer
);
483 emailMigration
.setSourceUsername(sourceUsername
);
484 emailMigration
.setSourcePassword(sourcePassword
);
485 emailMigration
.setTargetServer(targetServer
);
486 emailMigration
.setTargetUsername(targetUsername
);
487 emailMigration
.setTargetPassword(targetPassword
);
489 emailMigration
.process();