1 package org
.argeo
.app
.mail
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static java
.lang
.System
.Logger
.Level
.ERROR
;
5 import static org
.argeo
.app
.mail
.EmailUtils
.describe
;
7 import java
.io
.BufferedReader
;
8 import java
.io
.BufferedWriter
;
10 import java
.io
.FileOutputStream
;
11 import java
.io
.IOException
;
12 import java
.io
.InputStream
;
13 import java
.io
.OutputStream
;
14 import java
.lang
.System
.Logger
;
15 import java
.nio
.charset
.StandardCharsets
;
16 import java
.nio
.file
.Files
;
17 import java
.nio
.file
.Path
;
18 import java
.nio
.file
.StandardCopyOption
;
19 import java
.time
.Instant
;
20 import java
.util
.Date
;
21 import java
.util
.Enumeration
;
22 import java
.util
.Properties
;
24 import javax
.mail
.FetchProfile
;
25 import javax
.mail
.Folder
;
26 import javax
.mail
.Message
;
27 import javax
.mail
.MessagingException
;
28 import javax
.mail
.Multipart
;
29 import javax
.mail
.Session
;
30 import javax
.mail
.Store
;
31 import javax
.mail
.URLName
;
32 import javax
.mail
.internet
.InternetHeaders
;
33 import javax
.mail
.internet
.MimeBodyPart
;
34 import javax
.mail
.internet
.MimeMessage
;
35 import javax
.mail
.search
.HeaderTerm
;
36 import javax
.mail
.util
.SharedFileInputStream
;
38 import com
.sun
.mail
.imap
.IMAPFolder
;
39 import com
.sun
.mail
.mbox
.MboxFolder
;
40 import com
.sun
.mail
.mbox
.MboxMessage
;
42 /** Migrates emails from one storage to the another one. */
43 public class EmailMigration
{
44 private final static Logger logger
= System
.getLogger(EmailMigration
.class.getName());
46 // private String targetBaseDir;
48 private String sourceServer
;
49 private String sourceUsername
;
50 private String sourcePassword
;
52 private String targetServer
;
53 private String targetUsername
;
54 private String targetPassword
;
56 private boolean targetSupportDualTypeFolders
= true;
58 public void process() throws MessagingException
, IOException
{
59 // Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
61 Store sourceStore
= null;
63 Properties sourceProperties
= System
.getProperties();
64 sourceProperties
.setProperty("mail.store.protocol", "imaps");
66 Session sourceSession
= Session
.getInstance(sourceProperties
, null);
67 // session.setDebug(true);
68 sourceStore
= sourceSession
.getStore("imaps");
69 sourceStore
.connect(sourceServer
, sourceUsername
, sourcePassword
);
71 Folder defaultFolder
= sourceStore
.getDefaultFolder();
72 // migrateFolders(baseDir, defaultFolder);
74 // Always start with Inbox
75 // Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
76 // migrateFolder(baseDir, inboxFolder);
78 Properties targetProperties
= System
.getProperties();
79 targetProperties
.setProperty("mail.imap.starttls.enable", "true");
80 targetProperties
.setProperty("mail.imap.auth", "true");
82 Session targetSession
= Session
.getInstance(targetProperties
, null);
83 // session.setDebug(true);
84 Store targetStore
= targetSession
.getStore("imap");
85 targetStore
.connect(targetServer
, targetUsername
, targetPassword
);
87 // Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX);
88 // logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount());
89 // logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount());
91 migrateFolders(defaultFolder
, targetStore
);
93 if (sourceStore
!= null)
99 protected void migrateFolders(Folder sourceParentFolder
, Store targetStore
) throws MessagingException
, IOException
{
100 folders
: for (Folder sourceFolder
: sourceParentFolder
.list()) {
101 String sourceFolderName
= sourceFolder
.getName();
103 String sourceFolderFullName
= sourceFolder
.getFullName();
104 char sourceFolderSeparator
= sourceParentFolder
.getSeparator();
105 char targetFolderSeparator
= targetStore
.getDefaultFolder().getSeparator();
106 String targetFolderFullName
= sourceFolderFullName
.replace(sourceFolderSeparator
, targetFolderSeparator
);
109 if (sourceFolderFullName
.equals("[Gmail]")) {
110 migrateFolders(sourceFolder
, targetStore
);
113 if (sourceFolderFullName
.startsWith("[Gmail]")) {
114 String subFolderName
= null;
115 // Make it configurable
116 switch (sourceFolderName
) {
122 subFolderName
= "Sent";
126 targetFolderFullName
= subFolderName
== null ? sourceFolder
.getName() : subFolderName
;
129 // nature of the source folder
130 int messageCount
= (sourceFolder
.getType() & Folder
.HOLDS_MESSAGES
) != 0 ? sourceFolder
.getMessageCount()
132 boolean hasSubFolders
= (sourceFolder
.getType() & Folder
.HOLDS_FOLDERS
) != 0
133 ? sourceFolder
.list().length
!= 0
137 if (targetSupportDualTypeFolders
) {
138 targetFolder
= targetStore
.getFolder(targetFolderFullName
);
139 if (!targetFolder
.exists()) {
140 targetFolder
.create(Folder
.HOLDS_FOLDERS
| Folder
.HOLDS_MESSAGES
);
141 logger
.log(DEBUG
, "Created HOLDS_FOLDERS | HOLDS_MESSAGES folder " + targetFolder
.getFullName());
145 if (hasSubFolders
) {// has sub-folders
146 if (messageCount
== 0) {
147 targetFolder
= targetStore
.getFolder(targetFolderFullName
);
148 if (!targetFolder
.exists()) {
149 targetFolder
.create(Folder
.HOLDS_FOLDERS
);
150 logger
.log(DEBUG
, "Created HOLDS_FOLDERS folder " + targetFolder
.getFullName());
152 } else {// also has messages
153 Folder parentFolder
= targetStore
.getFolder(targetFolderFullName
);
154 if (!parentFolder
.exists()) {
155 parentFolder
.create(Folder
.HOLDS_FOLDERS
);
156 logger
.log(DEBUG
, "Created HOLDS_FOLDERS folder " + parentFolder
.getFullName());
158 String miscFullName
= targetFolderFullName
+ targetFolderSeparator
+ "_Misc";
159 targetFolder
= targetStore
.getFolder(miscFullName
);
160 if (!targetFolder
.exists()) {
161 targetFolder
.create(Folder
.HOLDS_MESSAGES
);
162 logger
.log(DEBUG
, "Created HOLDS_MESSAGES folder " + targetFolder
.getFullName());
165 } else {// no sub-folders
166 if (messageCount
== 0) { // empty
167 logger
.log(DEBUG
, "Skip empty folder " + targetFolderFullName
);
170 targetFolder
= targetStore
.getFolder(targetFolderFullName
);
171 if (!targetFolder
.exists()) {
172 targetFolder
.create(Folder
.HOLDS_MESSAGES
);
173 logger
.log(DEBUG
, "Created HOLDS_MESSAGES folder " + targetFolder
.getFullName());
178 if (messageCount
!= 0) {
180 targetFolder
.open(Folder
.READ_WRITE
);
182 long begin
= System
.currentTimeMillis();
183 sourceFolder
.open(Folder
.READ_ONLY
);
184 migrateFolder(sourceFolder
, targetFolder
);
185 long duration
= System
.currentTimeMillis() - begin
;
186 logger
.log(DEBUG
, targetFolderFullName
+ " - Migration of " + messageCount
+ " messages took "
187 + (duration
/ 1000) + " s (" + (duration
/ messageCount
) + " ms per message)");
189 sourceFolder
.close();
190 targetFolder
.close();
196 migrateFolders(sourceFolder
, targetStore
);
201 protected void migrateFoldersToFs(Path baseDir
, Folder sourceFolder
) throws MessagingException
, IOException
{
202 folders
: for (Folder folder
: sourceFolder
.list()) {
203 String folderName
= folder
.getName();
205 if ((folder
.getType() & Folder
.HOLDS_MESSAGES
) != 0) {
206 // Make it configurable
207 switch (folderName
) {
214 migrateFolderToFs(baseDir
, folder
);
216 if ((folder
.getType() & Folder
.HOLDS_FOLDERS
) != 0) {
217 migrateFoldersToFs(baseDir
.resolve(folder
.getName()), folder
);
222 protected void migrateFolderToFs(Path baseDir
, Folder sourceFolder
) throws MessagingException
, IOException
{
224 String folderName
= sourceFolder
.getName();
225 sourceFolder
.open(Folder
.READ_ONLY
);
227 Folder targetFolder
= null;
229 int messageCount
= sourceFolder
.getMessageCount();
230 logger
.log(DEBUG
, folderName
+ " - Message count : " + messageCount
);
231 if (messageCount
== 0)
233 // logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
235 boolean saveAsFiles
= false;
238 Message messages
[] = sourceFolder
.getMessages();
240 for (int i
= 0; i
< messages
.length
; ++i
) {
241 // logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
242 Message msg
= messages
[i
];
243 // String from = "unknown";
244 // if (msg.getReplyTo().length >= 1) {
245 // from = msg.getReplyTo()[0].toString();
246 // } else if (msg.getFrom().length >= 1) {
247 // from = msg.getFrom()[0].toString();
249 String subject
= msg
.getSubject();
250 Instant sentDate
= msg
.getSentDate().toInstant();
251 // logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
252 String fileName
= sentDate
+ " " + subject
;
253 Path file
= baseDir
.resolve(fileName
);
254 savePartsAsFiles(msg
.getContent(), file
);
257 long begin
= System
.currentTimeMillis();
258 targetFolder
= openMboxTargetFolder(sourceFolder
, baseDir
);
259 migrateFolder(sourceFolder
, targetFolder
);
260 long duration
= System
.currentTimeMillis() - begin
;
261 logger
.log(DEBUG
, folderName
+ " - Migration of " + messageCount
+ " messages took " + (duration
/ 1000)
262 + " s (" + (duration
/ messageCount
) + " ms per message)");
265 sourceFolder
.close();
266 if (targetFolder
!= null)
267 targetFolder
.close();
271 protected Folder
migrateFolder(Folder sourceFolder
, Folder targetFolder
) throws MessagingException
, IOException
{
272 String folderName
= targetFolder
.getName();
274 int lastSourceNumber
;
275 int currentTargetMessageCount
= targetFolder
.getMessageCount();
276 if (currentTargetMessageCount
!= 0) {
277 MimeMessage lastTargetMessage
= (MimeMessage
) targetFolder
.getMessage(currentTargetMessageCount
);
278 logger
.log(DEBUG
, folderName
+ " - Last target message " + describe(lastTargetMessage
));
279 Date lastTargetSent
= lastTargetMessage
.getReceivedDate();
280 Message
[] lastSourceMessage
= sourceFolder
281 .search(new HeaderTerm(EmailUtils
.MESSAGE_ID
, lastTargetMessage
.getMessageID()));
282 if (lastSourceMessage
.length
== 0)
283 throw new IllegalStateException("No message found with message ID " + lastTargetMessage
.getMessageID());
284 if (lastSourceMessage
.length
!= 1) {
285 for (Message msg
: lastSourceMessage
) {
286 logger
.log(ERROR
, "Message " + describe(msg
));
289 throw new IllegalStateException(
290 lastSourceMessage
.length
+ " messages found with received date " + lastTargetSent
.toInstant());
292 lastSourceNumber
= lastSourceMessage
[0].getMessageNumber();
294 lastSourceNumber
= 0;
296 logger
.log(DEBUG
, folderName
+ " - Last source message number " + lastSourceNumber
);
298 int countToRetrieve
= sourceFolder
.getMessageCount() - lastSourceNumber
;
300 FetchProfile fetchProfile
= new FetchProfile();
301 fetchProfile
.add(FetchProfile
.Item
.FLAGS
);
302 fetchProfile
.add(FetchProfile
.Item
.ENVELOPE
);
303 fetchProfile
.add(FetchProfile
.Item
.CONTENT_INFO
);
304 fetchProfile
.add(FetchProfile
.Item
.SIZE
);
305 if (sourceFolder
instanceof IMAPFolder
) {
306 // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
307 fetchProfile
.add(IMAPFolder
.FetchProfileItem
.HEADERS
);
308 fetchProfile
.add(IMAPFolder
.FetchProfileItem
.MESSAGE
);
312 int batchCount
= countToRetrieve
/ batchSize
;
313 if (countToRetrieve
% batchSize
!= 0)
314 batchCount
= batchCount
+ 1;
315 // int batchCount = 2; // for testing
316 for (int i
= 0; i
< batchCount
; i
++) {
317 long begin
= System
.currentTimeMillis();
319 int start
= lastSourceNumber
+ i
* batchSize
+ 1;
320 int end
= lastSourceNumber
+ (i
+ 1) * batchSize
;
321 if (end
>= (lastSourceNumber
+ countToRetrieve
+ 1))
322 end
= lastSourceNumber
+ countToRetrieve
;
323 Message
[] sourceMessages
= sourceFolder
.getMessages(start
, end
);
324 sourceFolder
.fetch(sourceMessages
, fetchProfile
);
325 // targetFolder.appendMessages(sourceMessages);
326 // sourceFolder.copyMessages(sourceMessages,targetFolder);
328 copyMessages(sourceMessages
, targetFolder
);
329 // copyMessagesToMbox(sourceMessages, targetFolder);
331 String describeLast
= describe(sourceMessages
[sourceMessages
.length
- 1]);
333 // if (i % 10 == 9) {
334 // free memory from fetched messages
335 sourceFolder
.close();
336 targetFolder
.close();
338 sourceFolder
.open(Folder
.READ_ONLY
);
339 targetFolder
.open(Folder
.READ_WRITE
);
340 // logger.log(DEBUG, "Open/close folder in order to free memory");
343 long duration
= System
.currentTimeMillis() - begin
;
344 logger
.log(DEBUG
, folderName
+ " - batch " + i
+ " took " + (duration
/ 1000) + " s, "
345 + (duration
/ (end
- start
+ 1)) + " ms per message. Last message " + describeLast
);
351 protected Folder
openMboxTargetFolder(Folder sourceFolder
, Path baseDir
) throws MessagingException
, IOException
{
352 String folderName
= sourceFolder
.getName();
353 if (sourceFolder
.getName().equals(EmailUtils
.INBOX_UPPER_CASE
))
354 folderName
= EmailUtils
.INBOX
;// Inbox
356 Path targetDir
= baseDir
;// .resolve("mbox");
357 Files
.createDirectories(targetDir
);
359 if (((sourceFolder
.getType() & Folder
.HOLDS_FOLDERS
) != 0) && sourceFolder
.list().length
!= 0) {
360 Path dir
= targetDir
.resolve(folderName
);
361 Files
.createDirectories(dir
);
362 targetPath
= dir
.resolve("_Misc");
364 targetPath
= targetDir
.resolve(folderName
);
366 if (!Files
.exists(targetPath
))
367 Files
.createFile(targetPath
);
368 URLName targetUrlName
= new URLName("mbox:" + targetPath
.toString());
369 Properties targetProperties
= new Properties();
370 // targetProperties.setProperty("mail.mime.address.strict", "false");
371 Session targetSession
= Session
.getDefaultInstance(targetProperties
);
372 Folder targetFolder
= targetSession
.getFolder(targetUrlName
);
373 targetFolder
.open(Folder
.READ_WRITE
);
378 protected void copyMessages(Message
[] sourceMessages
, Folder targetFolder
) throws MessagingException
{
379 targetFolder
.appendMessages(sourceMessages
);
382 protected void copyMessagesToMbox(Message
[] sourceMessages
, Folder targetFolder
)
383 throws MessagingException
, IOException
{
384 Message
[] targetMessages
= new Message
[sourceMessages
.length
];
385 for (int j
= 0; j
< sourceMessages
.length
; j
++) {
386 MimeMessage sourceMm
= (MimeMessage
) sourceMessages
[j
];
387 InternetHeaders ih
= new InternetHeaders();
388 for (Enumeration
<String
> e
= sourceMm
.getAllHeaderLines(); e
.hasMoreElements();) {
389 ih
.addHeaderLine(e
.nextElement());
391 Path tmpFileSource
= Files
.createTempFile("argeo-mbox-source", ".txt");
392 Path tmpFileTarget
= Files
.createTempFile("argeo-mbox-target", ".txt");
393 Files
.copy(sourceMm
.getRawInputStream(), tmpFileSource
, StandardCopyOption
.REPLACE_EXISTING
);
395 // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
396 // missing characters
397 try (BufferedReader reader
= Files
.newBufferedReader(tmpFileSource
, StandardCharsets
.ISO_8859_1
);
398 BufferedWriter writer
= Files
.newBufferedWriter(tmpFileTarget
, StandardCharsets
.ISO_8859_1
);) {
402 while ((line
= reader
.readLine()) != null) {
404 if (line
.startsWith("From ")) {
405 writer
.write(">" + line
);
407 "Fix line " + lineNumber
+ " in " + EmailUtils
.describe(sourceMm
) + ": " + line
);
413 } catch (IOException e
) {
414 logger
.log(ERROR
, "Error around line " + lineNumber
+ " of " + tmpFileSource
);
419 MboxMessage mboxMessage
= new MboxMessage((MboxFolder
) targetFolder
, ih
,
420 new SharedFileInputStream(tmpFileTarget
.toFile()), sourceMm
.getMessageNumber(),
421 EmailUtils
.getUnixFrom(sourceMm
), true);
422 targetMessages
[j
] = mboxMessage
;
425 Files
.delete(tmpFileSource
);
426 Files
.delete(tmpFileTarget
);
428 targetFolder
.appendMessages(targetMessages
);
432 /** Save body parts and attachments as plain files. */
433 protected void savePartsAsFiles(Object content
, Path fileBase
) throws IOException
, MessagingException
{
434 OutputStream out
= null;
435 InputStream in
= null;
437 if (content
instanceof Multipart
) {
438 Multipart multi
= ((Multipart
) content
);
439 int parts
= multi
.getCount();
440 for (int j
= 0; j
< parts
; ++j
) {
441 MimeBodyPart part
= (MimeBodyPart
) multi
.getBodyPart(j
);
442 if (part
.getContent() instanceof Multipart
) {
443 // part-within-a-part, do some recursion...
444 savePartsAsFiles(part
.getContent(), fileBase
);
446 String extension
= "";
447 if (part
.isMimeType("text/html")) {
450 if (part
.isMimeType("text/plain")) {
453 // Try to get the name of the attachment
454 extension
= part
.getDataHandler().getName();
457 String filename
= fileBase
+ "." + extension
;
458 System
.out
.println("... " + filename
);
459 out
= new FileOutputStream(new File(filename
));
460 in
= part
.getInputStream();
462 while ((k
= in
.read()) != -1) {
479 public void setSourceServer(String sourceServer
) {
480 this.sourceServer
= sourceServer
;
483 public void setSourceUsername(String sourceUsername
) {
484 this.sourceUsername
= sourceUsername
;
487 public void setSourcePassword(String sourcePassword
) {
488 this.sourcePassword
= sourcePassword
;
491 public void setTargetServer(String targetServer
) {
492 this.targetServer
= targetServer
;
495 public void setTargetUsername(String targetUsername
) {
496 this.targetUsername
= targetUsername
;
499 public void setTargetPassword(String targetPassword
) {
500 this.targetPassword
= targetPassword
;
503 public static void main(String args
[]) throws Exception
{
505 throw new IllegalArgumentException(
506 "usage: <source IMAP server> <source username> <source password> <target IMAP server> <target username> <target password>");
507 String sourceServer
= args
[0];
508 String sourceUsername
= args
[1];
509 String sourcePassword
= args
[2];
510 String targetServer
= args
[3];
511 String targetUsername
= args
[4];
512 String targetPassword
= args
[5];
514 EmailMigration emailMigration
= new EmailMigration();
515 emailMigration
.setSourceServer(sourceServer
);
516 emailMigration
.setSourceUsername(sourceUsername
);
517 emailMigration
.setSourcePassword(sourcePassword
);
518 emailMigration
.setTargetServer(targetServer
);
519 emailMigration
.setTargetUsername(targetUsername
);
520 emailMigration
.setTargetPassword(targetPassword
);
522 emailMigration
.process();