fix(Felamimail/Message): convert msg to eml

tests/tine20/Felamimail/Controller/MessageTest.php

@@ -642,6 +642,38 @@ public function testCheckOtherMails()
             }
         }
     }
+
+    /**
+     * validate fetching a complete message in 'other' dir and check its body
+     *
+     * howto:
+     * - copy mails to tests/tine20/Felamimail/files/other
+     * - add following header:
+     *      X-Tine20TestMessage: _filename_
+     * - run the test!
+     */
+    public function testCheckMsgMails()
+    {
+        $otherFilesDir = dirname(dirname(__FILE__)) . '/files';
+        if (file_exists($otherFilesDir)) {
+            foreach (new DirectoryIterator($otherFilesDir) as $item) {
+                $filename = $item->getFileName();
+                if ($item->isFile() && preg_match('/msg$/i', $filename)) {
+                    $fileName = '/' . $filename;
+                    echo "\nchecking message: " . $fileName . "\n";
+
+                    $this->_appendMessage($fileName, $this->_folder, []);
+                    $cachedMessage = $this->searchAndCacheMessage('l.kneschke@metaways.de', $this->_folder, true, 'To');
+                    $message = $this->_getController()->getCompleteMessage($cachedMessage);
+                    $plainMessage = $this->_getController()->getCompleteMessage($message, null, Zend_Mime::TYPE_TEXT);
+
+                    $this->assertTrue(! empty($message->body));
+                    $this->assertEquals(1, count($message->attachments));
+                    $this->assertEquals('moz-screenshot-83.png', $message->attachments[0]['filename']);
+                }
+            }
+        }
+    }
 
     /**
      * validate fetching a complete message
@@ -1770,7 +1802,7 @@ protected function _appendMessage($_filename, $_folder, $_replacements = array()
         } else {
             $message = fopen($filename, 'r');
         }
-        $this->_getController()->appendMessage($_folder, $message);
+        $this->_getController()->appendMessage($_folder, $message, null, $_filename);
     }
 
     /**

tine20/Felamimail/Controller/Message.php

@@ -115,13 +115,63 @@ public function checkFilterACL(Tinebase_Model_Filter_FilterGroup $_filter, $_act
      * append a new message to given folder
      *
      * @param string|Felamimail_Model_Folder $_folder id of target folder
-     * @param string|resource $_message full message content
+     * @param string|resource|Tinebase_Model_TempFile $_message full message content
      * @param array $_flags flags for new message
      */
-    public function appendMessage($_folder, $_message, $_flags = null)
+    public function appendMessage($_folder, $_message, $_flags = null, $_fileName = '')
     {
         $folder = ($_folder instanceof Felamimail_Model_Folder) ? $_folder : Felamimail_Controller_Folder::getInstance()->get($_folder);
-        $message = (is_resource($_message)) ? stream_get_contents($_message) : $_message;
+
+        if (str_contains($_fileName, '.msg')) {
+            $stream = $_message;
+            if ($_message instanceof Tinebase_Model_TempFile) {
+                $stream = fopen($_message->path, 'r');
+            }
+            try {
+                $documentFactory = new Pear\DocumentFactory();
+                $ole = $documentFactory->createFromStream($stream);
+                $mapiMessage = new Felamimail_MAPI_Message($ole);
+                $cacheId = sha1(self::class . $_fileName);
+                $message = $mapiMessage->parseMessage($cacheId);
+            } catch (Throwable $t) {
+                $message = 'Could not parse message: ' . $t->getMessage();
+                if (Tinebase_Core::isLogLevel(Zend_Log::NOTICE)) Tinebase_Core::getLogger()->notice(
+                    __METHOD__ . '::' . __LINE__ . ' ' . $message);
+                if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(
+                    __METHOD__ . '::' . __LINE__ . ' ' . $t->getTraceAsString());
+                throw new Tinebase_Exception_SystemGeneric($message);
+            } finally {
+                fclose($stream);
+            }
+            if ($message['body_content_type'] === 'text/html') {
+                $encoding = mb_detect_encoding($message['body']);
+                if (! $encoding) {
+                    $message['body'] = mb_convert_encoding($message['body'], 'UTF-8', 'ISO-8859-1');
+                }
+                $message->body = str_replace("\r", '', $message['body']);
+            }
+            $account = Felamimail_Controller_Account::getInstance()->get($folder->account_id);
+            $message->account_id = $folder->account_id;
+
+            $mail = Felamimail_Controller_Message_Send::getInstance()->createMailForSending($message, $account);
+
+            if ($message->sent instanceof Tinebase_DateTime) {
+                $dateTime  = new Zend_Date($message->sent);
+                $mail->setDate($dateTime);
+            }
+
+            $transport = new Felamimail_Transport();
+            $message = $transport->getRawMessage($mail);
+        } else {
+            $message = (is_resource($_message)) ? stream_get_contents($_message) : $_message;
+
+            if (!$_fileName || str_contains($_fileName, '.eml')) {
+                if ($_message instanceof Tinebase_Model_TempFile) {
+                    $message = file_get_contents($_message->path);
+                }
+            }
+        }
+
         $flags = ($_flags !== null) ? (array)$_flags : null;
 
         $imapBackend = $this->_getBackendAndSelectFolder(NULL, $folder);

tine20/Felamimail/Controller/Message/Send.php

@@ -1129,6 +1129,9 @@ protected function _getAttachmentPartByType(&$attachment, $_message)
             case 'tempfile':
                 $part = $this->_getTempFileAttachment($attachment);
                 break;
+            case 'stream':
+                $part = $this->_getStreamAttachment($attachment);
+                break;
             case 'messagepart':
             default:
                 $part = $this->_getMessagePartAttachment($attachment);
@@ -1165,6 +1168,8 @@ protected function _getAttachmentType($attachment, $_message)
         } elseif ($attachment instanceof Tinebase_Model_TempFile || isset($attachment['tempFile'])) {
             // tempfile last because we other attachment_types also have a tempfile
             return 'tempfile';
+        } elseif (isset($attachment['stream'])) {
+            return 'stream';
         }
 
         return null;
@@ -1384,6 +1389,28 @@ protected function _getTempFileAttachment(&$attachment)
         return $part;
     }
 
+    protected function _getStreamAttachment(&$attachment)
+    {
+        $part = new Zend_Mime_Part($attachment['stream']);
+        $encoding = Zend_Mime::ENCODING_BASE64;
+        if ($attachment['content-type']) {
+            $attachment['type'] = $attachment['content-type'];
+            $attachment['name'] = $attachment['filename'];
+
+            if ($attachment['type'] === Felamimail_Model_Message::CONTENT_TYPE_MESSAGE_RFC822) {
+                $encoding = Zend_Mime::ENCODING_8BIT;
+            }
+            if ($attachment['type'] === Felamimail_Model_Message::CONTENT_TYPE_MESSAGE_OUTLOOK) {
+                $attachment['name'] = $attachment['name'] . '.msg';
+            }
+            // not relevant?
+            $part->type = $attachment['content-type'];
+        }
+        $part->encoding = $encoding;
+
+        return $part;
+    }
+
     /**
      * @param mixed $attachment
      * @return null|Tinebase_Model_TempFile|Tinebase_Record_Interface

tine20/Felamimail/Frontend/Json.php

@@ -382,7 +382,7 @@ public function fileAttachments($id, $locations, $attachments = [], $model = 'Fe
     public function importMessage($targetFolderId, $tempFileId)
     {
         $tempFile = Tinebase_TempFile::getInstance()->get($tempFileId);
-        Felamimail_Controller_Message::getInstance()->appendMessage($targetFolderId, file_get_contents($tempFile->path));
+        Felamimail_Controller_Message::getInstance()->appendMessage($targetFolderId, $tempFile, null, $tempFile->name);
     }
 
     /**

tine20/Felamimail/MAPI/Message.php

@@ -27,7 +27,7 @@ class Felamimail_MAPI_Message extends BaseMessage
     public function parseMessage($cacheId)
     {
         $data = null;
-        
+
         if (Tinebase_Core::getCache()->test($cacheId) && ($messageData = Tinebase_Core::getCache()->load($cacheId) ?: null)) {
             $data = $messageData;
         }
@@ -53,11 +53,24 @@ public function parseMessage($cacheId)
                         }
                         break;
                     case 'from':
-                        $data['from_email'] = $this->properties['sender_email_address'];
-                        $data['from_name'] = $this->properties['sender_name'];
+                        $senderName = $this->properties['sender_name'];
+                        $senderAddr = $this->properties['sender_email_address'];
+                        $senderType = $this->properties['sender_addrtype'];
+
+                        $from = $senderAddr;
+                        if ($senderType !== 'SMTP') {
+                            $from = $this->properties['sender_smtp_address'] ??
+                                $this->properties['sender_representing_smtp_address'] ??
+                                sprintf('%s:%s', $senderType, $senderAddr);
+                        }
+                        $data['from_email'] = $from;
+                        $data['from_name'] = $senderName ?? '';
+
                         break;
                     //unset invalid fields for model message creation
                     case 'received':
+                        $received = explode(';', $headerValue[0]);
+                        $data['received'] = isset($received[1]) ? date("d-M-Y H:i:s O", strtotime($received[1])) : $headerValue[0];
                         break;
                     default:
                         $data[$headerKey] = $headerValue;
@@ -67,7 +80,7 @@ public function parseMessage($cacheId)
 
             $plainBody = $this->getBody();
             $htmlBody = $this->getBodyHTML();
-            
+
             if ($plainBody) {
                 $data['body'] = $plainBody;
                 $data['body_content_type'] = Zend_Mime::TYPE_TEXT;
@@ -78,37 +91,40 @@ public function parseMessage($cacheId)
                 $data['body_content_type'] = Zend_Mime::TYPE_HTML;
                 $data['body_content_type_of_body_property_of_this_record'] = Zend_Mime::TYPE_HTML;
             }
-
-            $data['attachments'] = [];
-
-            foreach ($this->getAttachments() as $id => $attachment) {
+        }
+        $data['content_type'] = Zend_Mime::TYPE_HTML;
+        $data['attachments'] = [];
+        $data['id'] = Tinebase_Record_Abstract::generateUID();
+        $attachments = $this->getAttachments();
+        if (count($attachments) > 0) {
+            foreach ($attachments as $id => $attachment) {
+                $stream = fopen("php://temp", 'r+');
+                $attachment->copyToStream($stream);
                 $data['attachments'][$id] = [
-                    'content-type'  => $attachment->getMimeType(),
-                    'filename'      => $attachment->getFilename(),
-                    'partId'        => $id,
-                    'data'          => $attachment->getData(),
+                    'content-type' => $attachment->getMimeType(),
+                    'filename' => $attachment->getFilename(),
+                    'partId' => $id,
+                    'stream' => $stream
                 ];
             }
+            $data['has_attachment'] = true;
         }
-
-        //attachment can not be fetched from cache , so we always create attachment from attachment data;
-        if (count($data['attachments']) > 0) {
+
+        Tinebase_Core::getCache()->save($data, $cacheId);
+
+        if (count($attachments) > 0) {
             $msg = new ZBateson\MailMimeParser\Message();
-
-            foreach ($data['attachments'] as $id => $attachment) {
-                $msg->addAttachmentPart($attachment['data'], $attachment['content-type'], $attachment['filename']);
-            }
-            foreach ($msg->getAllAttachmentParts() as $id => $attachment) {
+            foreach ($data['attachments'] as $id => &$attachmentData) {
+                $msg->addAttachmentPart($attachmentData['stream'], $attachmentData['content-type'],$attachmentData['filename']);
+                $result = $msg->getAttachmentPart($id);
                 /** @var GuzzleHttp\Psr7\Stream $partStream */
-                $partStream = $attachment->getStream();
-                $data['attachments'][$id]['size']           = $partStream->getSize();
-                $data['attachments'][$id]['contentstream']  = $attachment->getContentStream();
-                $data['attachments'][$id]['stream']         = $attachment->getBinaryContentResourceHandle();
+                $partStream = $result->getStream();
+                $attachmentData['size']           = $partStream->getSize();
+                $attachmentData['contentstream']  = $result->getContentStream();
+                $attachmentData['stream']         = $result->getBinaryContentResourceHandle();
             }
-            $data['has_attachment'] = true;
         }
-        // write message data to cache
-        Tinebase_Core::getCache()->save($data, $cacheId);
+
         return new Felamimail_Model_Message($data);
     }
 
@@ -136,30 +152,19 @@ protected function translatePropertyHeaders()
         foreach ($transport as $header) {
             $rawHeaders->add($header);
         }
-
-        // sender        
-        $senderType = $this->properties['sender_addrtype'];
-        if ($senderType == 'SMTP') {
-            $rawHeaders->set('From', $this->getSender());
-        }
-        elseif (!$rawHeaders->has('From')) {
-            if ($from = $this->getSender()) {
-                $rawHeaders->set('From', $from);
-            }
-        }
-
+
         // recipients
         $recipients = $this->getRecipients();
-        
+
         // fix duplicated content
         foreach (['to', 'cc', 'bcc'] as $type) {
             $rawHeaders->unset($type);
         }
-        
+
         foreach ($recipients as $r) {
             $rawHeaders->add($r->getType(), (string)$r);
         }
-        
+
         // subject - preference to msg properties
         if ($this->properties['subject']) {
             $rawHeaders->set('Subject', $this->properties['subject']);

tine20/Felamimail/Model/Message.php

@@ -47,6 +47,8 @@ class Felamimail_Model_Message extends Tinebase_Record_Abstract implements Tineb
      */
     const CONTENT_TYPE_MESSAGE_RFC822 = 'message/rfc822';
 
+    const CONTENT_TYPE_MESSAGE_OUTLOOK = 'Microsoft Office Outlook Message';
+
     /**
      * content type html
      *

tine20/Felamimail/js/GridPanel.js

@@ -505,8 +505,8 @@ Tine.Felamimail.GridPanel = Ext.extend(Tine.widgets.grid.GridPanel, {
         }
 
         const exts = _.uniq(_.map(fileSelector.files, file => {return file.name.split('.').pop()}));
-        if (exts.length !== 1 || exts[0] !== 'eml') {
-            return Ext.Msg.alert(this.app.i18n._('Wrong File Type'), this.app.i18n._('Files of type eml allowed only.'));
+        if (exts.length !== 1 || !['eml', 'msg'].includes(exts[0])) {
+            return Ext.Msg.alert(this.app.i18n._('Wrong File Type'), this.app.i18n._('Files of type eml and msg allowed only.'));
         }
 
         this.importMask = new Ext.LoadMask(this.grid.getEl(), { msg: i18n._('Importing Messages...') });

tine20/Felamimail/translations/de.po

@@ -1994,8 +1994,8 @@ msgstr "Sie müssen einen Zielordner für die Nachrichten auswählen."
 msgid "Wrong File Type"
 msgstr "Falscher Dateityp"
 
-msgid "Files of type eml allowed only."
-msgstr "Nur Dateien vom Typ eml sind erlaubt."
+msgid "Files of type eml and msg allowed only."
+msgstr "Nur Dateien vom Typ eml und msg sind erlaubt."
 
 msgid "Importing Messages..."
 msgstr "Importiere Nachrichten..."