1 package org
.argeo
.slc
.mail
;
3 import static java
.lang
.System
.Logger
.Level
.DEBUG
;
4 import static java
.lang
.System
.Logger
.Level
.ERROR
;
5 import static org
.argeo
.slc
.mail
.EmailUtils
.describe
;
7 import java
.io
.BufferedReader
;
8 import java
.io
.BufferedWriter
;
10 import java
.io
.FileOutputStream
;
11 import java
.io
.IOException
;
12 import java
.io
.InputStream
;
13 import java
.io
.OutputStream
;
14 import java
.lang
.System
.Logger
;
15 import java
.nio
.charset
.StandardCharsets
;
16 import java
.nio
.file
.Files
;
17 import java
.nio
.file
.Path
;
18 import java
.nio
.file
.StandardCopyOption
;
19 import java
.time
.Instant
;
20 import java
.util
.Date
;
21 import java
.util
.Enumeration
;
22 import java
.util
.Properties
;
24 import javax
.mail
.FetchProfile
;
25 import javax
.mail
.Folder
;
26 import javax
.mail
.Message
;
27 import javax
.mail
.MessagingException
;
28 import javax
.mail
.Multipart
;
29 import javax
.mail
.Session
;
30 import javax
.mail
.Store
;
31 import javax
.mail
.URLName
;
32 import javax
.mail
.internet
.InternetHeaders
;
33 import javax
.mail
.internet
.MimeBodyPart
;
34 import javax
.mail
.internet
.MimeMessage
;
35 import javax
.mail
.search
.HeaderTerm
;
36 import javax
.mail
.util
.SharedFileInputStream
;
38 import com
.sun
.mail
.imap
.IMAPFolder
;
39 import com
.sun
.mail
.mbox
.MboxFolder
;
40 import com
.sun
.mail
.mbox
.MboxMessage
;
42 /** Migrates emails from one storage to the another one. */
43 public class EmailMigration
{
44 private final static Logger logger
= System
.getLogger(EmailMigration
.class.getName());
46 // private String targetBaseDir;
48 private String sourceServer
;
49 private String sourceUsername
;
50 private String sourcePassword
;
52 private String targetServer
;
53 private String targetUsername
;
54 private String targetPassword
;
56 public void process() throws MessagingException
, IOException
{
57 // Path baseDir = Paths.get(targetBaseDir).resolve(sourceUsername).resolve("mbox");
59 Store sourceStore
= null;
61 Properties sourceProperties
= System
.getProperties();
62 sourceProperties
.setProperty("mail.store.protocol", "imaps");
64 Session sourceSession
= Session
.getInstance(sourceProperties
, null);
65 // session.setDebug(true);
66 sourceStore
= sourceSession
.getStore("imaps");
67 sourceStore
.connect(sourceServer
, sourceUsername
, sourcePassword
);
69 Folder defaultFolder
= sourceStore
.getDefaultFolder();
70 // migrateFolders(baseDir, defaultFolder);
72 // Always start with Inbox
73 // Folder inboxFolder = sourceStore.getFolder(EmailUtils.INBOX);
74 // migrateFolder(baseDir, inboxFolder);
76 Properties targetProperties
= System
.getProperties();
77 targetProperties
.setProperty("mail.imap.starttls.enable", "true");
78 targetProperties
.setProperty("mail.imap.auth", "true");
80 Session targetSession
= Session
.getInstance(targetProperties
, null);
81 // session.setDebug(true);
82 Store targetStore
= targetSession
.getStore("imap");
83 targetStore
.connect(targetServer
, targetUsername
, targetPassword
);
85 // Folder targetFolder = targetStore.getFolder(EmailUtils.INBOX);
86 // logger.log(DEBUG, "Source message count " + inboxFolder.getMessageCount());
87 // logger.log(DEBUG, "Target message count " + targetFolder.getMessageCount());
89 migrateFolders(defaultFolder
, targetStore
);
91 if (sourceStore
!= null)
97 protected void migrateFolders(Folder sourceFolder
, Store targetStore
) throws MessagingException
, IOException
{
98 folders
: for (Folder folder
: sourceFolder
.list()) {
99 String folderName
= folder
.getName();
101 String folderFullName
= folder
.getFullName();
104 if (folderFullName
.equals("[Gmail]")) {
105 migrateFolders(folder
, targetStore
);
108 if (folderFullName
.startsWith("[Gmail]")) {
109 // Make it configurable
110 switch (folderName
) {
118 folderFullName
= folder
.getName();
121 int messageCount
= (folder
.getType() & Folder
.HOLDS_MESSAGES
) != 0 ? folder
.getMessageCount() : 0;
122 boolean hasSubFolders
= (folder
.getType() & Folder
.HOLDS_FOLDERS
) != 0 ? folder
.list().length
!= 0 : false;
124 if (hasSubFolders
) {// has sub-folders
125 if (messageCount
== 0) {
126 targetFolder
= targetStore
.getFolder(folderFullName
);
127 if (!targetFolder
.exists()) {
128 targetFolder
.create(Folder
.HOLDS_FOLDERS
);
129 logger
.log(DEBUG
, "Created HOLDS_FOLDERS folder " + targetFolder
.getFullName());
131 } else {// also has messages
132 Folder parentFolder
= targetStore
.getFolder(folderFullName
);
133 if (!parentFolder
.exists()) {
134 parentFolder
.create(Folder
.HOLDS_FOLDERS
);
135 logger
.log(DEBUG
, "Created HOLDS_FOLDERS folder " + parentFolder
.getFullName());
137 String miscFullName
= folderFullName
+ "/_Misc";
138 targetFolder
= targetStore
.getFolder(miscFullName
);
139 if (!targetFolder
.exists()) {
140 targetFolder
.create(Folder
.HOLDS_MESSAGES
);
141 logger
.log(DEBUG
, "Created HOLDS_MESSAGES folder " + targetFolder
.getFullName());
144 } else {// no sub-folders
145 if (messageCount
== 0) { // empty
146 logger
.log(DEBUG
, "Skip empty folder " + folderFullName
);
149 targetFolder
= targetStore
.getFolder(folderFullName
);
150 if (!targetFolder
.exists()) {
151 targetFolder
.create(Folder
.HOLDS_MESSAGES
);
152 logger
.log(DEBUG
, "Created HOLDS_MESSAGES folder " + targetFolder
.getFullName());
156 if (messageCount
!= 0) {
158 targetFolder
.open(Folder
.READ_WRITE
);
160 long begin
= System
.currentTimeMillis();
161 folder
.open(Folder
.READ_ONLY
);
162 migrateFolder(folder
, targetFolder
);
163 long duration
= System
.currentTimeMillis() - begin
;
164 logger
.log(DEBUG
, folderFullName
+ " - Migration of " + messageCount
+ " messages took "
165 + (duration
/ 1000) + " s (" + (duration
/ messageCount
) + " ms per message)");
168 targetFolder
.close();
174 migrateFolders(folder
, targetStore
);
179 protected void migrateFoldersToFs(Path baseDir
, Folder sourceFolder
) throws MessagingException
, IOException
{
180 folders
: for (Folder folder
: sourceFolder
.list()) {
181 String folderName
= folder
.getName();
183 if ((folder
.getType() & Folder
.HOLDS_MESSAGES
) != 0) {
184 // Make it configurable
185 switch (folderName
) {
192 migrateFolderToFs(baseDir
, folder
);
194 if ((folder
.getType() & Folder
.HOLDS_FOLDERS
) != 0) {
195 migrateFoldersToFs(baseDir
.resolve(folder
.getName()), folder
);
200 protected void migrateFolderToFs(Path baseDir
, Folder sourceFolder
) throws MessagingException
, IOException
{
202 String folderName
= sourceFolder
.getName();
203 sourceFolder
.open(Folder
.READ_ONLY
);
205 Folder targetFolder
= null;
207 int messageCount
= sourceFolder
.getMessageCount();
208 logger
.log(DEBUG
, folderName
+ " - Message count : " + messageCount
);
209 if (messageCount
== 0)
211 // logger.log(DEBUG, folderName + " - Unread Messages : " + sourceFolder.getUnreadMessageCount());
213 boolean saveAsFiles
= false;
216 Message messages
[] = sourceFolder
.getMessages();
218 for (int i
= 0; i
< messages
.length
; ++i
) {
219 // logger.log(DEBUG, "MESSAGE #" + (i + 1) + ":");
220 Message msg
= messages
[i
];
221 // String from = "unknown";
222 // if (msg.getReplyTo().length >= 1) {
223 // from = msg.getReplyTo()[0].toString();
224 // } else if (msg.getFrom().length >= 1) {
225 // from = msg.getFrom()[0].toString();
227 String subject
= msg
.getSubject();
228 Instant sentDate
= msg
.getSentDate().toInstant();
229 // logger.log(DEBUG, "Saving ... " + subject + " from " + from + " (" + sentDate + ")");
230 String fileName
= sentDate
+ " " + subject
;
231 Path file
= baseDir
.resolve(fileName
);
232 savePartsAsFiles(msg
.getContent(), file
);
236 long begin
= System
.currentTimeMillis();
237 targetFolder
= openMboxTargetFolder(sourceFolder
, baseDir
);
238 migrateFolder(sourceFolder
, targetFolder
);
239 long duration
= System
.currentTimeMillis() - begin
;
240 logger
.log(DEBUG
, folderName
+ " - Migration of " + messageCount
+ " messages took " + (duration
/ 1000)
241 + " s (" + (duration
/ messageCount
) + " ms per message)");
244 sourceFolder
.close();
245 if (targetFolder
!= null)
246 targetFolder
.close();
250 protected Folder
migrateFolder(Folder sourceFolder
, Folder targetFolder
) throws MessagingException
, IOException
{
251 String folderName
= targetFolder
.getName();
253 int lastSourceNumber
;
254 int currentTargetMessageCount
= targetFolder
.getMessageCount();
255 if (currentTargetMessageCount
!= 0) {
256 MimeMessage lastTargetMessage
= (MimeMessage
) targetFolder
.getMessage(currentTargetMessageCount
);
257 logger
.log(DEBUG
, folderName
+ " - Last target message " + describe(lastTargetMessage
));
258 Date lastTargetSent
= lastTargetMessage
.getReceivedDate();
259 Message
[] lastSourceMessage
= sourceFolder
260 .search(new HeaderTerm(EmailUtils
.MESSAGE_ID
, lastTargetMessage
.getMessageID()));
261 if (lastSourceMessage
.length
== 0)
262 throw new IllegalStateException("No message found with message ID " + lastTargetMessage
.getMessageID());
263 if (lastSourceMessage
.length
!= 1) {
264 for (Message msg
: lastSourceMessage
) {
265 logger
.log(ERROR
, "Message " + describe(msg
));
268 throw new IllegalStateException(
269 lastSourceMessage
.length
+ " messages found with received date " + lastTargetSent
.toInstant());
271 lastSourceNumber
= lastSourceMessage
[0].getMessageNumber();
273 lastSourceNumber
= 0;
275 logger
.log(DEBUG
, folderName
+ " - Last source message number " + lastSourceNumber
);
277 int countToRetrieve
= sourceFolder
.getMessageCount() - lastSourceNumber
;
279 FetchProfile fetchProfile
= new FetchProfile();
280 fetchProfile
.add(FetchProfile
.Item
.FLAGS
);
281 fetchProfile
.add(FetchProfile
.Item
.ENVELOPE
);
282 fetchProfile
.add(FetchProfile
.Item
.CONTENT_INFO
);
283 fetchProfile
.add(FetchProfile
.Item
.SIZE
);
284 if (sourceFolder
instanceof IMAPFolder
) {
285 // IMAPFolder sourceImapFolder = (IMAPFolder) sourceFolder;
286 fetchProfile
.add(IMAPFolder
.FetchProfileItem
.HEADERS
);
287 fetchProfile
.add(IMAPFolder
.FetchProfileItem
.MESSAGE
);
291 int batchCount
= countToRetrieve
/ batchSize
;
292 if (countToRetrieve
% batchSize
!= 0)
293 batchCount
= batchCount
+ 1;
294 // int batchCount = 2; // for testing
295 for (int i
= 0; i
< batchCount
; i
++) {
296 long begin
= System
.currentTimeMillis();
298 int start
= lastSourceNumber
+ i
* batchSize
+ 1;
299 int end
= lastSourceNumber
+ (i
+ 1) * batchSize
;
300 if (end
>= (lastSourceNumber
+ countToRetrieve
+ 1))
301 end
= lastSourceNumber
+ countToRetrieve
;
302 Message
[] sourceMessages
= sourceFolder
.getMessages(start
, end
);
303 sourceFolder
.fetch(sourceMessages
, fetchProfile
);
304 // targetFolder.appendMessages(sourceMessages);
305 // sourceFolder.copyMessages(sourceMessages,targetFolder);
307 copyMessages(sourceMessages
, targetFolder
);
308 // copyMessagesToMbox(sourceMessages, targetFolder);
310 String describeLast
= describe(sourceMessages
[sourceMessages
.length
- 1]);
312 // if (i % 10 == 9) {
313 // free memory from fetched messages
314 sourceFolder
.close();
315 targetFolder
.close();
317 sourceFolder
.open(Folder
.READ_ONLY
);
318 targetFolder
.open(Folder
.READ_WRITE
);
319 // logger.log(DEBUG, "Open/close folder in order to free memory");
322 long duration
= System
.currentTimeMillis() - begin
;
323 logger
.log(DEBUG
, folderName
+ " - batch " + i
+ " took " + (duration
/ 1000) + " s, "
324 + (duration
/ (end
- start
+ 1)) + " ms per message. Last message " + describeLast
);
330 protected Folder
openMboxTargetFolder(Folder sourceFolder
, Path baseDir
) throws MessagingException
, IOException
{
331 String folderName
= sourceFolder
.getName();
332 if (sourceFolder
.getName().equals(EmailUtils
.INBOX_UPPER_CASE
))
333 folderName
= EmailUtils
.INBOX
;// Inbox
335 Path targetDir
= baseDir
;// .resolve("mbox");
336 Files
.createDirectories(targetDir
);
338 if (((sourceFolder
.getType() & Folder
.HOLDS_FOLDERS
) != 0) && sourceFolder
.list().length
!= 0) {
339 Path dir
= targetDir
.resolve(folderName
);
340 Files
.createDirectories(dir
);
341 targetPath
= dir
.resolve("_Misc");
343 targetPath
= targetDir
.resolve(folderName
);
345 if (!Files
.exists(targetPath
))
346 Files
.createFile(targetPath
);
347 URLName targetUrlName
= new URLName("mbox:" + targetPath
.toString());
348 Properties targetProperties
= new Properties();
349 // targetProperties.setProperty("mail.mime.address.strict", "false");
350 Session targetSession
= Session
.getDefaultInstance(targetProperties
);
351 Folder targetFolder
= targetSession
.getFolder(targetUrlName
);
352 targetFolder
.open(Folder
.READ_WRITE
);
357 protected void copyMessages(Message
[] sourceMessages
, Folder targetFolder
) throws MessagingException
{
358 targetFolder
.appendMessages(sourceMessages
);
361 protected void copyMessagesToMbox(Message
[] sourceMessages
, Folder targetFolder
)
362 throws MessagingException
, IOException
{
363 Message
[] targetMessages
= new Message
[sourceMessages
.length
];
364 for (int j
= 0; j
< sourceMessages
.length
; j
++) {
365 MimeMessage sourceMm
= (MimeMessage
) sourceMessages
[j
];
366 InternetHeaders ih
= new InternetHeaders();
367 for (Enumeration
<String
> e
= sourceMm
.getAllHeaderLines(); e
.hasMoreElements();) {
368 ih
.addHeaderLine(e
.nextElement());
370 Path tmpFileSource
= Files
.createTempFile("argeo-mbox-source", ".txt");
371 Path tmpFileTarget
= Files
.createTempFile("argeo-mbox-target", ".txt");
372 Files
.copy(sourceMm
.getRawInputStream(), tmpFileSource
, StandardCopyOption
.REPLACE_EXISTING
);
374 // we use ISO_8859_1 because it is more robust than US_ASCII with regard to
375 // missing characters
376 try (BufferedReader reader
= Files
.newBufferedReader(tmpFileSource
, StandardCharsets
.ISO_8859_1
);
377 BufferedWriter writer
= Files
.newBufferedWriter(tmpFileTarget
, StandardCharsets
.ISO_8859_1
);) {
381 while ((line
= reader
.readLine()) != null) {
383 if (line
.startsWith("From ")) {
384 writer
.write(">" + line
);
386 "Fix line " + lineNumber
+ " in " + EmailUtils
.describe(sourceMm
) + ": " + line
);
392 } catch (IOException e
) {
393 logger
.log(ERROR
, "Error around line " + lineNumber
+ " of " + tmpFileSource
);
398 MboxMessage mboxMessage
= new MboxMessage((MboxFolder
) targetFolder
, ih
,
399 new SharedFileInputStream(tmpFileTarget
.toFile()), sourceMm
.getMessageNumber(),
400 EmailUtils
.getUnixFrom(sourceMm
), true);
401 targetMessages
[j
] = mboxMessage
;
404 Files
.delete(tmpFileSource
);
405 Files
.delete(tmpFileTarget
);
407 targetFolder
.appendMessages(targetMessages
);
411 /** Save body parts and attachments as plain files. */
412 protected void savePartsAsFiles(Object content
, Path fileBase
) throws IOException
, MessagingException
{
413 OutputStream out
= null;
414 InputStream in
= null;
416 if (content
instanceof Multipart
) {
417 Multipart multi
= ((Multipart
) content
);
418 int parts
= multi
.getCount();
419 for (int j
= 0; j
< parts
; ++j
) {
420 MimeBodyPart part
= (MimeBodyPart
) multi
.getBodyPart(j
);
421 if (part
.getContent() instanceof Multipart
) {
422 // part-within-a-part, do some recursion...
423 savePartsAsFiles(part
.getContent(), fileBase
);
425 String extension
= "";
426 if (part
.isMimeType("text/html")) {
429 if (part
.isMimeType("text/plain")) {
432 // Try to get the name of the attachment
433 extension
= part
.getDataHandler().getName();
436 String filename
= fileBase
+ "." + extension
;
437 System
.out
.println("... " + filename
);
438 out
= new FileOutputStream(new File(filename
));
439 in
= part
.getInputStream();
441 while ((k
= in
.read()) != -1) {
458 public void setSourceServer(String sourceServer
) {
459 this.sourceServer
= sourceServer
;
462 public void setSourceUsername(String sourceUsername
) {
463 this.sourceUsername
= sourceUsername
;
466 public void setSourcePassword(String sourcePassword
) {
467 this.sourcePassword
= sourcePassword
;
470 public void setTargetServer(String targetServer
) {
471 this.targetServer
= targetServer
;
474 public void setTargetUsername(String targetUsername
) {
475 this.targetUsername
= targetUsername
;
478 public void setTargetPassword(String targetPassword
) {
479 this.targetPassword
= targetPassword
;
482 public static void main(String args
[]) throws Exception
{
484 throw new IllegalArgumentException(
485 "usage: <source IMAP server> <source username> <source password> <target IMAP server> <target username> <target password>");
486 String sourceServer
= args
[0];
487 String sourceUsername
= args
[1];
488 String sourcePassword
= args
[2];
489 String targetServer
= args
[3];
490 String targetUsername
= args
[4];
491 String targetPassword
= args
[5];
493 EmailMigration emailMigration
= new EmailMigration();
494 emailMigration
.setSourceServer(sourceServer
);
495 emailMigration
.setSourceUsername(sourceUsername
);
496 emailMigration
.setSourcePassword(sourcePassword
);
497 emailMigration
.setTargetServer(targetServer
);
498 emailMigration
.setTargetUsername(targetUsername
);
499 emailMigration
.setTargetPassword(targetPassword
);
501 emailMigration
.process();