From 1baf6687ab3a95f8a94f3a0249fe7b3fee58007d Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 1 Apr 2016 23:45:41 +0200 Subject: [PATCH 01/68] Fixed receipt ack not including group participant --- .../layers/protocol_acks/protocolentities/ack_outgoing.py | 6 +++--- .../protocol_receipts/protocolentities/receipt_incoming.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py b/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py index b9439455b..e62e4bc68 100644 --- a/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py +++ b/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py @@ -11,15 +11,15 @@ class OutgoingAckProtocolEntity(AckProtocolEntity): ''' - def __init__(self, _id, _class, _type, _to, _participant = None): + def __init__(self, _id, _class, _type, to, participant = None): super(OutgoingAckProtocolEntity, self).__init__(_id, _class) - self.setOutgoingData(_type, _to, _participant) + self.setOutgoingData(_type, to, participant) def setOutgoingData(self, _type, _to, _participant): self._type = _type self._to = _to self._participant = _participant - + def toProtocolTreeNode(self): node = super(OutgoingAckProtocolEntity, self).toProtocolTreeNode() if self._type: diff --git a/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py b/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py index 6592d67c7..181f9b24a 100644 --- a/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py +++ b/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py @@ -100,7 +100,7 @@ def __str__(self): return out def ack(self): - return OutgoingAckProtocolEntity(self.getId(), "receipt", self.getType(), self.getFrom()) + return OutgoingAckProtocolEntity(self.getId(), "receipt", self.getType(), self.getFrom(), participant = self.participant) @staticmethod def fromProtocolTreeNode(node): From 328b9d22b353aa01d318c54828f50d4be92ed720 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 1 Apr 2016 23:46:02 +0200 Subject: [PATCH 02/68] Keep track of connection status in network layer Fixes multiple redundant disconnect events being emitted --- yowsup/layers/network/layer.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/yowsup/layers/network/layer.py b/yowsup/layers/network/layer.py index fdb5b6adb..44b7f8e69 100644 --- a/yowsup/layers/network/layer.py +++ b/yowsup/layers/network/layer.py @@ -18,8 +18,14 @@ class YowNetworkLayer(YowLayer, asyncore.dispatcher_with_send): PROP_ENDPOINT = "org.openwhatsapp.yowsup.prop.endpoint" PROP_NET_READSIZE = "org.openwhatsapp.yowsup.prop.net.readSize" + STATE_DISCONNECTED = 0 + STATE_CONNECTING = 1 + STATE_CONNECTED = 2 + STATE_DISCONNECTING = 3 + def __init__(self): asyncore.dispatcher.__init__(self) + self.state = self.__class__.STATE_DISCONNECTED YowLayer.__init__(self) self.interface = YowNetworkLayerInterface(self) httpProxy = HttpProxy.getFromEnviron() @@ -33,18 +39,19 @@ def onConnect(): proxyHandler = httpProxy.handler() proxyHandler.onConnect = onConnect self.proxyHandler = proxyHandler - + @EventCallback(EVENT_STATE_CONNECT) def onConnect(self, ev): self.createConnection() return True - + @EventCallback(EVENT_STATE_DISCONNECT) def onDisconnect(self, ev): self.destroyConnection(ev.getArg("reason")) - return True + return True def createConnection(self): + self.state = self.__class__.STATE_CONNECTING self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.out_buffer = bytearray() endpoint = self.getProp(self.__class__.PROP_ENDPOINT) @@ -53,15 +60,20 @@ def createConnection(self): logger.debug("HttpProxy connect: %s:%d" % endpoint) self.proxyHandler.connect(self, endpoint) else: - self.connect(endpoint) + try: + self.connect(endpoint) + except OSError as e: + self.handle_close(e) def destroyConnection(self, reason = None): + self.state = self.__class__.STATE_DISCONNECTING self.handle_close(reason or "Requested") def getStatus(self): return self.connected def handle_connect(self): + self.state = self.__class__.STATE_CONNECTED self.connected = True if self.proxyHandler != None: logger.debug("HttpProxy handle connect") @@ -70,10 +82,12 @@ def handle_connect(self): self.emitEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_CONNECTED)) def handle_close(self, reason = "Connection Closed"): - self.connected = False - logger.debug("Disconnected, reason: %s" % reason) - self.emitEvent(YowLayerEvent(self.__class__.EVENT_STATE_DISCONNECTED, reason = reason, detached=True)) - self.close() + if self.state != self.__class__.STATE_DISCONNECTED: + self.state = self.__class__.STATE_DISCONNECTED + self.connected = False + logger.debug("Disconnected, reason: %s" % reason) + self.emitEvent(YowLayerEvent(self.__class__.EVENT_STATE_DISCONNECTED, reason = reason, detached=True)) + self.close() def handle_error(self): raise From 3db22d1506b75c2a737c3a55b8bd21a82b094da9 Mon Sep 17 00:00:00 2001 From: Jlguardi Date: Fri, 8 Apr 2016 09:56:53 +0200 Subject: [PATCH 03/68] Fixed #1464, mime_types included into egg and used pkg_resources --- setup.py | 1 + yowsup/common/tools.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4f88da5cb..2e2fd6292 100755 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ #long_description=long_description, packages= find_packages(), include_package_data=True, + data_files = [('yowsup/common', ['yowsup/common/mime.types'])], platforms='any', #test_suite='', classifiers = [ diff --git a/yowsup/common/tools.py b/yowsup/common/tools.py index dbeda2d6f..7a83919eb 100644 --- a/yowsup/common/tools.py +++ b/yowsup/common/tools.py @@ -9,6 +9,7 @@ import hashlib import os.path, mimetypes from .optionalmodules import PILOptionalModule, FFVideoOptionalModule +from pkg_resources import resource_string logger = logging.getLogger(__name__) @@ -150,10 +151,9 @@ def generatePreviewFromImage(image): return preview class MimeTools: - MIME_FILE = os.path.join(os.path.dirname(__file__), 'mime.types') + MIME_FILE = resource_string(__name__, 'mime.types') mimetypes.init() # Load default mime.types mimetypes.init([MIME_FILE]) # Append whatsapp mime.types - decode_hex = codecs.getdecoder("hex_codec") @staticmethod def getMIME(filepath): From daf4516692bc1023d9116b9c19a9aed14fb5dd1e Mon Sep 17 00:00:00 2001 From: Jlguardi Date: Sat, 16 Apr 2016 11:09:51 +0200 Subject: [PATCH 04/68] Fixed pip install --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..edcf88f22 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include yowsup/common/mime.types From 05d3e43f3961b3d83df3c177271e7631f838838d Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Wed, 20 Apr 2016 13:52:10 +0200 Subject: [PATCH 05/68] Fixed wrong arg name, refs #1522 --- yowsup/common/optionalmodules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yowsup/common/optionalmodules.py b/yowsup/common/optionalmodules.py index 09d1b8e64..7ec4a8fe7 100644 --- a/yowsup/common/optionalmodules.py +++ b/yowsup/common/optionalmodules.py @@ -40,7 +40,7 @@ def __init__(self, failMessage = None, require = False): class AxolotlOptionalModule(OptionalModule): def __init__(self, failMessage = None, require = False): super(AxolotlOptionalModule, self).__init__("axolotl", - failmessage=failMessage, + failMessage=failMessage, require=require) if __name__ == "__main__": From 195cbf4b6a27032970b365e164b4f5a0d2abe85b Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 12:07:08 +0200 Subject: [PATCH 06/68] Autorecover connection initial commit --- yowsup/layers/auth/layer_authentication.py | 11 ++-- .../layers/auth/protocolentities/__init__.py | 3 +- .../auth/protocolentities/stream_error.py | 66 +++++++++++++++++++ .../auth/protocolentities/stream_error_ack.py | 22 ------- .../protocolentities/stream_error_conflict.py | 35 ---------- yowsup/layers/interface/interface.py | 28 ++++++++ 6 files changed, 101 insertions(+), 64 deletions(-) create mode 100644 yowsup/layers/auth/protocolentities/stream_error.py delete mode 100644 yowsup/layers/auth/protocolentities/stream_error_ack.py delete mode 100644 yowsup/layers/auth/protocolentities/stream_error_conflict.py diff --git a/yowsup/layers/auth/layer_authentication.py b/yowsup/layers/auth/layer_authentication.py index 5b6aedb02..bd73cd516 100644 --- a/yowsup/layers/auth/layer_authentication.py +++ b/yowsup/layers/auth/layer_authentication.py @@ -8,6 +8,7 @@ from .protocolentities import * from yowsup.common.tools import StorageTools from .layer_interface_authentication import YowAuthenticationProtocolLayerInterface +from .protocolentities import StreamErrorProtocolEntity import base64 class YowAuthenticationProtocolLayer(YowProtocolLayer): EVENT_LOGIN = "org.openwhatsapp.yowsup.event.auth.login" @@ -82,12 +83,12 @@ def handleChallenge(self, node): self._sendResponse(nodeEntity.getNonce()) def handleStreamError(self, node): - if node.getChild("text"): - nodeEntity = StreamErrorConflictProtocolEntity.fromProtocolTreeNode(node) - elif node.getChild("ack"): - nodeEntity = StreamErrorAckProtocolEntity.fromProtocolTreeNode(node) - else: + nodeEntity = StreamErrorProtocolEntity.fromProtocolTreeNode(node) + errorType = nodeEntity.getErrorType() + + if not errorType: raise AuthError("Unhandled stream:error node:\n%s" % node) + self.toUpper(nodeEntity) ##senders diff --git a/yowsup/layers/auth/protocolentities/__init__.py b/yowsup/layers/auth/protocolentities/__init__.py index b78c067a0..bc4db2e29 100644 --- a/yowsup/layers/auth/protocolentities/__init__.py +++ b/yowsup/layers/auth/protocolentities/__init__.py @@ -4,5 +4,4 @@ from .stream_features import StreamFeaturesProtocolEntity from .success import SuccessProtocolEntity from .failure import FailureProtocolEntity -from .stream_error_conflict import StreamErrorConflictProtocolEntity -from .stream_error_ack import StreamErrorAckProtocolEntity +from .stream_error import StreamErrorProtocolEntity diff --git a/yowsup/layers/auth/protocolentities/stream_error.py b/yowsup/layers/auth/protocolentities/stream_error.py new file mode 100644 index 000000000..c44bec387 --- /dev/null +++ b/yowsup/layers/auth/protocolentities/stream_error.py @@ -0,0 +1,66 @@ +from yowsup.structs import ProtocolEntity, ProtocolTreeNode +class StreamErrorProtocolEntity(ProtocolEntity): + TYPE_CONFLICT = "conflict" + ''' + + + Replaced by new connection + + ''' + + TYPE_ACK = "ack" + ''' + + + + ''' + + TYPE_XML_NOT_WELL_FORMED = "xml-not-well-formed" + ''' + + + + + ''' + + TYPES = (TYPE_CONFLICT, TYPE_ACK, TYPE_XML_NOT_WELL_FORMED) + + def __init__(self, data = None): + super(StreamErrorProtocolEntity, self).__init__("stream:error") + data = data or {} + self.setErrorData(data) + + def setErrorData(self, data): + self.data = data + + def getErrorData(self): + return self.data + + def getErrorType(self): + for k in self.data.keys(): + if k in self.__class__.TYPES: + return k + + def __str__(self): + out = "Stream Error type: %s\n" % self.getErrorType() + out += self.getErrorData() + out += "\n" + + return out + + def toProtocolTreeNode(self): + node = super(StreamErrorProtocolEntity, self).toProtocolTreeNode() + type = self.getErrorType() + node.addChild(ProtocolTreeNode(type)) + if type == self.__class__.TYPE_CONFLICT and "text" in self.data: + node.addChild(ProtocolTreeNode("text", data=self.data["text"])) + + return node + + + @staticmethod + def fromProtocolTreeNode(protocolTreeNode): + data = {} + for child in protocolTreeNode.getAllChildren(): + data[child.tag] = child.data + return StreamErrorProtocolEntity(data) diff --git a/yowsup/layers/auth/protocolentities/stream_error_ack.py b/yowsup/layers/auth/protocolentities/stream_error_ack.py deleted file mode 100644 index 054c473ec..000000000 --- a/yowsup/layers/auth/protocolentities/stream_error_ack.py +++ /dev/null @@ -1,22 +0,0 @@ -from yowsup.structs import ProtocolEntity, ProtocolTreeNode -class StreamErrorAckProtocolEntity(ProtocolEntity): - ''' - - - - ''' - def __init__(self): - super(StreamErrorAckProtocolEntity, self).__init__("stream:error") - - def toProtocolTreeNode(self): - node = super(StreamErrorAckProtocolEntity, self).toProtocolTreeNode() - node.addChild(ProtocolTreeNode("ack")) - return node - - def __str__(self): - out = "Ack Stream Error\n" - return out - - @staticmethod - def fromProtocolTreeNode(node): - return StreamErrorAckProtocolEntity() diff --git a/yowsup/layers/auth/protocolentities/stream_error_conflict.py b/yowsup/layers/auth/protocolentities/stream_error_conflict.py deleted file mode 100644 index 6d6a0535c..000000000 --- a/yowsup/layers/auth/protocolentities/stream_error_conflict.py +++ /dev/null @@ -1,35 +0,0 @@ -from yowsup.structs import ProtocolEntity, ProtocolTreeNode -class StreamErrorConflictProtocolEntity(ProtocolEntity): - ''' - - - Replaced by new connection - - ''' - def __init__(self, text = None): - super(StreamErrorConflictProtocolEntity, self).__init__("stream:error") - self.setText(text) - - def setText(self, text = None): - self.text = text or '' - - def getText(self): - return self.text - - def __str__(self): - out = "Conflict Stream Error\n" - if self.text: - out += "Text: %s\n" % self.getText() - - return out - - def toProtocolTreeNode(self): - node = super(StreamErrorConflictProtocolEntity, self).toProtocolTreeNode() - node.addChild(ProtocolTreeNode("conflict")) - node.addChild(ProtocolTreeNode("text", data=self.text)) - return node - - - @staticmethod - def fromProtocolTreeNode(node): - return StreamErrorConflictProtocolEntity(node.getChild("text").getData()) diff --git a/yowsup/layers/interface/interface.py b/yowsup/layers/interface/interface.py index 57f8a52f7..9674e37d1 100644 --- a/yowsup/layers/interface/interface.py +++ b/yowsup/layers/interface/interface.py @@ -4,7 +4,11 @@ from yowsup.layers.auth import YowAuthenticationProtocolLayer from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity from yowsup.layers.protocol_acks.protocolentities import IncomingAckProtocolEntity +from yowsup.layers.network.layer import YowNetworkLayer +from yowsup.layers import EventCallback import inspect +import logging +logger = logging.getLogger(__name__) class ProtocolEntityCallback(object): def __init__(self, entityType): @@ -17,8 +21,11 @@ def __call__(self, fn): class YowInterfaceLayer(YowLayer): + PROP_RECONNECT_ON_STREAM_ERR = "org.openwhatsapp.yowsup.prop.interface.reconnect_on_stream_error" + def __init__(self): super(YowInterfaceLayer, self).__init__() + self.reconnect = False self.entity_callbacks = {} self.iqRegistry = {} # self.receiptsRegistry = {} @@ -101,5 +108,26 @@ def receive(self, entity): else: self.toUpper(entity) + @ProtocolEntityCallback("stream:error") + def onStreamError(self, streamErrorEntity): + logger.error(streamErrorEntity) + if self.getProp(self.__class__.PROP_RECONNECT_ON_STREAM_ERR, True): + logger.info("Initiating reconnect") + self.reconnect = True + self.disconnect() + else: + logger.warn("No reconnecting because property %s is not set" % self.__class__.PROP_RECONNECT_ON_STREAM_ERR) + self.toUpper(streamErrorEntity) + + @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) + def onConnected(self, yowLayerEvent): + self.reconnect = False + + @EventCallback(YowNetworkLayer.EVENT_STATE_DISCONNECTED) + def onDisconnected(self, yowLayerEvent): + if self.reconnect: + self.reconnect = False + self.connect() + def __str__(self): return "Interface Layer" From b5404ed9a226c51a14b13b3565b613035f3aa0e4 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 12:23:08 +0100 Subject: [PATCH 07/68] New decode funcs for wa1.6 --- yowsup/layers/coder/decoder.py | 135 ++++++++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 20 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 3093b1ad1..849676580 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -1,5 +1,6 @@ from yowsup.structs import ProtocolTreeNode import math +import binascii class ReadDecoder: def __init__(self, tokenDictionary): self.streamStarted = False; @@ -13,7 +14,7 @@ def getProtocolTreeNode(self, data): return self.streamStart(data) return self.nextTreeInternal(data) - def _getToken(self, index, data): + def getToken(self, index, data): token = self.tokenDictionary.getToken(index) if not token: index = self.readInt8(data) @@ -23,6 +24,13 @@ def _getToken(self, index, data): return token + def getTokenDouble(self, n, n2): + pos = n2 + n * 256 + ret = "" + token = self.tokenDictionary.getToken(pos, True) + if not token: + raise ValueError("Invalid token %s" % pos) + def streamStart(self, data): self.streamStarted = True tag = data.pop(0) @@ -32,7 +40,7 @@ def streamStart(self, data): if tag != 1: if tag == 236: tag = data.pop(0) + 237 - token = self._getToken(tag, data)#self.tokenDictionary.getToken(tag) + token = self.getToken(tag, data)#self.tokenDictionary.getToken(tag) raise Exception("expecting STREAM_START in streamStart, instead got token: %s" % token) attribCount = (size - 2 + size % 2) / 2 self.readAttributes(attribCount, data) @@ -57,12 +65,77 @@ def readNibble(self, data): raise Exception("Bad nibble %s" % dec) return string + def readPacked8(self, n, data): + size = self.readInt8(data) + remove = 0 + if (size & 0x80) != 0 and n == 251: + remove = 1 + size = size & 0x7F + text = bytearray(self.readArray(size, data)) + data = binascii.hexlify(text) + out = "" + for i in range(0, len(data)): + char = chr(data[i]) if type(data[i]) is int else data[i] #python2/3 compat + val = ord(binascii.unhexlify("0%s" % char)) + if i == size - 1 and val > 11 and n != 251: continue + out += chr(self.unpackByte(n, val)) + + return out[0:len(out) - remove] + + def unpackByte(self, n, n2): + if n == 251: + return self.unpackHex(n2) + if n == 255: + return self.unpackNibble(n2) + raise ValueError("bad packed type %s" % n) + + def unpackHex(self, n): + if n in range(0, 10): + return n + 48 + if n in range(10, 16): + return 65 + (n - 10) + + raise ValueError("bad hex %s" % n) + + def unpackNibble(self, n): + if n in range(0, 10): + return n + 48 + if n in (10, 11): + return 45 + (n - 10) + raise ValueError("bad nibble %s" % n) + + + + def readHeader(self, data, offset = 0): + ret = 0 + if len(data) >= (3 + offset): + b0 = data[offset] + b1 = data[offset + 1] + b2 = data[offset + 2] + ret = b0 + (b1 << 16) + (b2 << 8) + + return ret + + def peekInt8(self, data, offset = 0): + value = 0 + if len(data) >= (1 + offset): + value = data[offset] + return value + def readInt8(self, data): + return data.pop(0); - def readInt8(self,data): - return data.pop(0); + def peekInt16(self, data, offset = 0): + value = 0 + if len(data) >= (2 + offset): + intTop = data[offset] + intBot = data[offset + 1] + value = (intTop << 8) + intBot + + return value + def readInt16(self, data): intTop = data.pop(0) @@ -73,6 +146,17 @@ def readInt16(self, data): else: return "" + + def peekInt24(self, data, offset = 0): + value = 0 + if len(data) >= (3 + offset): + int1 = data[offset] + int2 = data[offset + 1] + int3 = data[offset + 2] + value = (int1 << 16) + (int2 << 8) + (int3 << 0) + + return value + def readInt24(self,data): int1 = data.pop(0) int2 = data.pop(0) @@ -81,6 +165,9 @@ def readInt24(self,data): return value + + + def readListSize(self,token, data): size = 0 if token == 0: @@ -106,27 +193,17 @@ def readAttributes(self, attribCount, data): def readString(self,token, data): - if token == -1: raise Exception("-1 token in readString") - if token > 2 and token < 245: - return self._getToken(token, data) + if 2 < token < 236: + return self.getToken(token, data) if token == 0: return None - - if token == 252: - size8 = self.readInt8(data) - buf8 = self.readArray(size8, data) - return "".join(map(chr, buf8)) - - - if token == 253: - size24 = self.readInt24(data) - buf24 = self.readArray(size24, data) - return "".join(map(chr, buf24)) + if token in (236, 237, 238, 239): + return self.getTokenDouble(token - 236, self.readInt8(data)) if token == 250: user = self.readString(data.pop(0), data) @@ -136,8 +213,26 @@ def readString(self,token, data): if server is not None: return server raise Exception("readString couldn't reconstruct jid") - elif token == 255: - return self.readNibble(data) + + if token in (251, 255): + return self.readPacked8(token, data) + + + if token == 252: + size8 = self.readInt8(data) + buf8 = self.readArray(size8, data) + return "".join(map(chr, buf8)) + + if token == 253: + size20 = self.readInt20(data) + buf20 = self.readArray(size20, data) + return "".join(map(chr, buf20)) + + if token == 254: + size31 = self.readInt31() + buf31 = self.readArray(size31, data) + return "".join(map(chr, buf31)) + raise Exception("readString couldn't match token "+str(token)) From 1270d1dfbc49aa869cfd13f391cc9c01bb0cabb6 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 14:33:39 +0100 Subject: [PATCH 08/68] Decoder initial port --- yowsup/layers/coder/decoder.py | 81 ++++++++++++++++------------------ 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 849676580..52fa4521d 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -105,7 +105,6 @@ def unpackNibble(self, n): raise ValueError("bad nibble %s" % n) - def readHeader(self, data, offset = 0): ret = 0 if len(data) >= (3 + offset): @@ -116,27 +115,9 @@ def readHeader(self, data, offset = 0): return ret - def peekInt8(self, data, offset = 0): - value = 0 - if len(data) >= (1 + offset): - value = data[offset] - - return value - def readInt8(self, data): return data.pop(0); - - def peekInt16(self, data, offset = 0): - value = 0 - if len(data) >= (2 + offset): - intTop = data[offset] - intBot = data[offset + 1] - value = (intTop << 8) + intBot - - return value - - def readInt16(self, data): intTop = data.pop(0) intBot = data.pop(0) @@ -146,27 +127,25 @@ def readInt16(self, data): else: return "" + def readInt20(self, data): + int1 = self.data.pop(0) + int2 = self.data.pop(1) + int3 = self.data.pop(2) + return ((int1 & 0xF) << 16) + (int2 << 8) + (int3 << 0) - def peekInt24(self, data, offset = 0): - value = 0 - if len(data) >= (3 + offset): - int1 = data[offset] - int2 = data[offset + 1] - int3 = data[offset + 2] - value = (int1 << 16) + (int2 << 8) + (int3 << 0) - - return value - - def readInt24(self,data): + def readInt24(self, data): int1 = data.pop(0) int2 = data.pop(0) int3 = data.pop(0) value = (int1 << 16) + (int2 << 8) + (int3 << 0) return value - - - + def readInt31(self, data): + data.pop(0) + int1 = data.pop(0) + int2 = data.pop(0) + int3 = data.pop(0) + return (int1 << 24) | (int1 << 16) | int2 << 8 | int3 def readListSize(self,token, data): size = 0 @@ -244,15 +223,16 @@ def readArray(self, length, data): return out def nextTreeInternal(self, data): - b = data.pop(0) + size = self.readListSize(self.readInt8(data)) + token = self.readInt8(data) + if token == 1: + token = self.readInt8(data) - size = self.readListSize(b, data) - b = data.pop(0) - if b == 2: + if token == 2: return None + tag = self.readString(token, data) - tag = self.readString(b, data) if size == 0 or tag is None: raise ValueError("nextTree sees 0 list or null tag") @@ -261,12 +241,25 @@ def nextTreeInternal(self, data): if size % 2 ==1: return ProtocolTreeNode(tag, attribs) - b = data.pop(0) - - if self.isListTag(b): - return ProtocolTreeNode(tag,attribs,self.readList(b, data)) - - return ProtocolTreeNode(tag, attribs, None, self.readString(b, data)) + read2 = self.readInt8(data) + + nodeData = None + if self.isListTag(read2): + nodeData = self.readList(read2, data) + if read2 == 252: + size = self.readInt8(data) + nodeData = self.readArray(size, data) + if read2 == 253: + size = self.readInt20(data) + nodeData = self.readArray(size, data) + if read2 == 254: + size = self.readInt31(data) + nodeData = self.readArray(size, data) + if read2 in (255, 251): + nodeData = self.readPacked8(read2, data) + + nodeData = nodeData or self.readString(read2, data) + return ProtocolTreeNode(tag, attribs, None, nodeData) def readList(self,token, data): size = self.readListSize(token, data) From 429bb1414a0843261254a4845b367c8f2520c3fa Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 14:52:13 +0100 Subject: [PATCH 09/68] updated token dictionary --- yowsup/layers/coder/tokendictionary.py | 960 +++++++++++++------------ 1 file changed, 499 insertions(+), 461 deletions(-) diff --git a/yowsup/layers/coder/tokendictionary.py b/yowsup/layers/coder/tokendictionary.py index 7b711e17a..ac4edd7e2 100644 --- a/yowsup/layers/coder/tokendictionary.py +++ b/yowsup/layers/coder/tokendictionary.py @@ -2,469 +2,508 @@ class TokenDictionary: def __init__(self): self.dictionary = [ - "", - "", - "", - "account", - "ack", - "action", - "active", - "add", - "after", - "all", - "allow", - "apple", - "auth", - "author", - "available", - "bad-protocol", - "bad-request", - "before", - "body", - "broadcast", - "cancel", - "category", - "challenge", - "chat", - "clean", - "code", - "composing", - "config", - "contacts", - "count", - "create", - "creation", - "debug", - "default", - "delete", - "delivery", - "delta", - "deny", - "digest", - "dirty", - "duplicate", - "elapsed", - "enable", - "encoding", - "error", - "event", - "expiration", - "expired", - "fail", - "failure", - "false", - "favorites", - "feature", - "features", - "feature-not-implemented", - "field", - "first", - "free", - "from", - "g.us", - "get", - "google", - "group", - "groups", - "groups_v2", - "http://etherx.jabber.org/streams", - "http://jabber.org/protocol/chatstates", - "ib", - "id", - "image", - "img", - "index", - "internal-server-error", - "ip", - "iq", - "item-not-found", - "item", - "jabber:iq:last", - "jabber:iq:privacy", - "jabber:x:event", - "jid", - "kind", - "last", - "leave", - "list", - "max", - "mechanism", - "media", - "message_acks", - "message", - "method", - "microsoft", - "missing", - "modify", - "mute", - "name", - "nokia", - "none", - "not-acceptable", - "not-allowed", - "not-authorized", - "notification", - "notify", - "off", - "offline", - "order", - "owner", - "owning", - "p_o", - "p_t", - "paid", - "participant", - "participants", - "participating", - "paused", - "picture", - "pin", - "ping", - "platform", - "port", - "presence", - "preview", - "probe", - "prop", - "props", - "query", - "raw", - "read", - "readreceipts", - "reason", - "receipt", - "relay", - "remote-server-timeout", - "remove", - "request", - "required", - "resource-constraint", - "resource", - "response", - "result", - "retry", - "rim", - "s_o", - "s_t", - "s.us", - "s.whatsapp.net", - "seconds", - "server-error", - "server", - "service-unavailable", - "set", - "show", - "silent", - "stat", - "status", - "stream:error", - "stream:features", - "subject", - "subscribe", - "success", - "sync", - "t", - "text", - "timeout", - "timestamp", - "to", - "true", - "type", - "unavailable", - "unsubscribe", - "uri", - "url", - "urn:ietf:params:xml:ns:xmpp-sasl", - "urn:ietf:params:xml:ns:xmpp-stanzas", - "urn:ietf:params:xml:ns:xmpp-streams", - "urn:xmpp:ping", - "urn:xmpp:whatsapp:account", - "urn:xmpp:whatsapp:dirty", - "urn:xmpp:whatsapp:mms", - "urn:xmpp:whatsapp:push", - "urn:xmpp:whatsapp", - "user", - "user-not-found", - "value", - "version", - "w:g", - "w:p:r", - "w:p", - "w:profile:picture", - "w", - "wait", - "WAUTH-2", - "xmlns:stream", - "xmlns", - "1", - "chatstate", - "crypto", - "phash", - "enc", - "class", - "off_cnt", - "w:g2", - "promote", - "demote", - "creator", - "Bell.caf", - "Boing.caf", - "Glass.caf", - "Harp.caf", - "TimePassing.caf", - "Tri-tone.caf", - "Xylophone.caf", - "background", - "backoff", - "chunked", - "context", - "full", - "in", - "interactive", - "out", - "registration", - "sid", - "urn:xmpp:whatsapp:sync", - "flt", - "s16", - "u8", - "adpcm", - "amrnb", - "amrwb", - "mp3", - "pcm", - "qcelp", - "wma", - "h263", - "h264", - "jpeg" + '', + '', + '', + 'account', + 'ack', + 'action', + 'active', + 'add', + 'after', + 'all', + 'allow', + 'apple', + 'audio', + 'auth', + 'author', + 'available', + 'bad-protocol', + 'bad-request', + 'before', + 'bits', + 'body', + 'broadcast', + 'cancel', + 'category', + 'challenge', + 'chat', + 'clean', + 'code', + 'composing', + 'config', + 'contacts', + 'count', + 'create', + 'creation', + 'debug', + 'default', + 'delete', + 'delivery', + 'delta', + 'deny', + 'digest', + 'dirty', + 'duplicate', + 'elapsed', + 'enable', + 'encoding', + 'encrypt', + 'error', + 'event', + 'expiration', + 'expired', + 'fail', + 'failure', + 'false', + 'favorites', + 'feature', + 'features', + 'feature-not-implemented', + 'field', + 'file', + 'filehash', + 'first', + 'free', + 'from', + 'g.us', + 'gcm', + 'get', + 'google', + 'group', + 'groups', + 'groups_v2', + 'http://etherx.jabber.org/streams', + 'http://jabber.org/protocol/chatstates', + 'ib', + 'id', + 'image', + 'img', + 'index', + 'internal-server-error', + 'ip', + 'iq', + 'item-not-found', + 'item', + 'jabber:iq:last', + 'jabber:iq:privacy', + 'jabber:x:event', + 'jid', + 'kind', + 'last', + 'leave', + 'list', + 'max', + 'mechanism', + 'media', + 'message_acks', + 'message', + 'method', + 'microsoft', + 'mimetype', + 'missing', + 'modify', + 'msg', + 'mute', + 'name', + 'nokia', + 'none', + 'not-acceptable', + 'not-allowed', + 'not-authorized', + 'notification', + 'notify', + 'off', + 'offline', + 'order', + 'owner', + 'owning', + 'p_o', + 'p_t', + 'paid', + 'participant', + 'participants', + 'participating', + 'paused', + 'picture', + 'pin', + 'ping', + 'pkmsg', + 'platform', + 'port', + 'presence', + 'preview', + 'probe', + 'prop', + 'props', + 'qcount', + 'query', + 'raw', + 'read', + 'readreceipts', + 'reason', + 'receipt', + 'relay', + 'remote-server-timeout', + 'remove', + 'request', + 'required', + 'resource-constraint', + 'resource', + 'response', + 'result', + 'retry', + 'rim', + 's_o', + 's_t', + 's.us', + 's.whatsapp.net', + 'seconds', + 'server-error', + 'server', + 'service-unavailable', + 'set', + 'show', + 'silent', + 'size', + 'skmsg', + 'stat', + 'state', + 'status', + 'stream:error', + 'stream:features', + 'subject', + 'subscribe', + 'success', + 'sync', + 't', + 'text', + 'timeout', + 'timestamp', + 'tizen', + 'to', + 'true', + 'type', + 'unavailable', + 'unsubscribe', + 'upgrade', + 'uri', + 'url', + 'urn:ietf:params:xml:ns:xmpp-sasl', + 'urn:ietf:params:xml:ns:xmpp-stanzas', + 'urn:ietf:params:xml:ns:xmpp-streams', + 'urn:xmpp:ping', + 'urn:xmpp:whatsapp:account', + 'urn:xmpp:whatsapp:dirty', + 'urn:xmpp:whatsapp:mms', + 'urn:xmpp:whatsapp:push', + 'urn:xmpp:whatsapp', + 'user', + 'user-not-found', + 'v', + 'value', + 'version', + 'voip', + 'w:g', + 'w:p:r', + 'w:p', + 'w:profile:picture', + 'w', + 'wait', + 'WAUTH-2', + 'xmlns:stream', + 'xmlns', + '1', + 'chatstate', + 'crypto', + 'phash', + 'enc', + 'class', + 'off_cnt', + 'w:g2', + 'promote', + 'demote', + 'creator', + 'background', + 'backoff', + 'chunked', + 'context', + 'full', + 'in', + 'interactive', + 'out', + 'registration', + 'sid', + 'urn:xmpp:whatsapp:sync', + 'flt', + 's16', + 'u8', ] self.secondaryDictionary = [ - "mpeg4", - "wmv", - "audio/3gpp", - "audio/aac", - "audio/amr", - "audio/mp4", - "audio/mpeg", - "audio/ogg", - "audio/qcelp", - "audio/wav", - "audio/webm", - "audio/x-caf", - "audio/x-ms-wma", - "image/gif", - "image/jpeg", - "image/png", - "video/3gpp", - "video/avi", - "video/mp4", - "video/mpeg", - "video/quicktime", - "video/x-flv", - "video/x-ms-asf", - "302", - "400", - "401", - "402", - "403", - "404", - "405", - "406", - "407", - "409", - "410", - "500", - "501", - "503", - "504", - "abitrate", - "acodec", - "app_uptime", - "asampfmt", - "asampfreq", - "audio", - "clear", - "conflict", - "conn_no_nna", - "cost", - "currency", - "duration", - "extend", - "file", - "fps", - "g_notify", - "g_sound", - "gcm", - "gone", - "google_play", - "hash", - "height", - "invalid", - "jid-malformed", - "latitude", - "lc", - "lg", - "live", - "location", - "log", - "longitude", - "max_groups", - "max_participants", - "max_subject", - "mimetype", - "mode", - "napi_version", - "normalize", - "orighash", - "origin", - "passive", - "password", - "played", - "policy-violation", - "pop_mean_time", - "pop_plus_minus", - "price", - "pricing", - "redeem", - "Replaced by new connection", - "resume", - "signature", - "size", - "sound", - "source", - "system-shutdown", - "username", - "vbitrate", - "vcard", - "vcodec", - "video", - "width", - "xml-not-well-formed", - "checkmarks", - "image_max_edge", - "image_max_kbytes", - "image_quality", - "ka", - "ka_grow", - "ka_shrink", - "newmedia", - "library", - "caption", - "forward", - "c0", - "c1", - "c2", - "c3", - "clock_skew", - "cts", - "k0", - "k1", - "login_rtt", - "m_id", - "nna_msg_rtt", - "nna_no_off_count", - "nna_offline_ratio", - "nna_push_rtt", - "no_nna_con_count", - "off_msg_rtt", - "on_msg_rtt", - "stat_name", - "sts", - "suspect_conn", - "lists", - "self", - "qr", - "web", - "w:b", - "recipient", - "w:stats", - "forbidden", - "aurora.m4r", - "bamboo.m4r", - "chord.m4r", - "circles.m4r", - "complete.m4r", - "hello.m4r", - "input.m4r", - "keys.m4r", - "note.m4r", - "popcorn.m4r", - "pulse.m4r", - "synth.m4r", - "filehash", - "max_list_recipients", - "en-AU", - "en-GB", - "es-MX", - "pt-PT", - "zh-Hans", - "zh-Hant", - "relayelection", - "relaylatency", - "interruption", - "Apex.m4r", - "Beacon.m4r", - "Bulletin.m4r", - "By The Seaside.m4r", - "Chimes.m4r", - "Circuit.m4r", - "Constellation.m4r", - "Cosmic.m4r", - "Crystals.m4r", - "Hillside.m4r", - "Illuminate.m4r", - "Night Owl.m4r", - "Opening.m4r", - "Playtime.m4r", - "Presto.m4r", - "Radar.m4r", - "Radiate.m4r", - "Ripples.m4r", - "Sencha.m4r", - "Signal.m4r", - "Silk.m4r", - "Slow Rise.m4r", - "Stargaze.m4r", - "Summit.m4r", - "Twinkle.m4r", - "Uplift.m4r", - "Waves.m4r", - "voip", - "eligible", - "upgrade", - "planned", - "current", - "future", - "disable", - "expire", - "start", - "stop", - "accuracy", - "speed", - "bearing", - "recording", - "encrypt", - "key", - "identity", - "w:gp2", - "admin", - "locked", - "unlocked", - "new", - "battery", - "archive", - "adm", - "plaintext_size", - "compressed_size", - "delivered", - "msg", - "pkmsg", - "everyone", - "v", - "transport", - "call-id" + 'adpcm', + 'amrnb', + 'amrwb', + 'mp3', + 'pcm', + 'qcelp', + 'wma', + 'h263', + 'h264', + 'jpeg', + 'mpeg4', + 'wmv', + 'audio/3gpp', + 'audio/aac', + 'audio/amr', + 'audio/mp4', + 'audio/mpeg', + 'audio/ogg', + 'audio/qcelp', + 'audio/wav', + 'audio/webm', + 'audio/x-caf', + 'audio/x-ms-wma', + 'image/gif', + 'image/jpeg', + 'image/png', + 'video/3gpp', + 'video/avi', + 'video/mp4', + 'video/mpeg', + 'video/quicktime', + 'video/x-flv', + 'video/x-ms-asf', + '302', + '400', + '401', + '402', + '403', + '404', + '405', + '406', + '407', + '409', + '410', + '500', + '501', + '503', + '504', + 'abitrate', + 'acodec', + 'app_uptime', + 'asampfmt', + 'asampfreq', + 'clear', + 'conflict', + 'conn_no_nna', + 'cost', + 'currency', + 'duration', + 'extend', + 'fps', + 'g_notify', + 'g_sound', + 'gone', + 'google_play', + 'hash', + 'height', + 'invalid', + 'jid-malformed', + 'latitude', + 'lc', + 'lg', + 'live', + 'location', + 'log', + 'longitude', + 'max_groups', + 'max_participants', + 'max_subject', + 'mode', + 'napi_version', + 'normalize', + 'orighash', + 'origin', + 'passive', + 'password', + 'played', + 'policy-violation', + 'pop_mean_time', + 'pop_plus_minus', + 'price', + 'pricing', + 'redeem', + 'Replaced by new connection', + 'resume', + 'signature', + 'sound', + 'source', + 'system-shutdown', + 'username', + 'vbitrate', + 'vcard', + 'vcodec', + 'video', + 'width', + 'xml-not-well-formed', + 'checkmarks', + 'image_max_edge', + 'image_max_kbytes', + 'image_quality', + 'ka', + 'ka_grow', + 'ka_shrink', + 'newmedia', + 'library', + 'caption', + 'forward', + 'c0', + 'c1', + 'c2', + 'c3', + 'clock_skew', + 'cts', + 'k0', + 'k1', + 'login_rtt', + 'm_id', + 'nna_msg_rtt', + 'nna_no_off_count', + 'nna_offline_ratio', + 'nna_push_rtt', + 'no_nna_con_count', + 'off_msg_rtt', + 'on_msg_rtt', + 'stat_name', + 'sts', + 'suspect_conn', + 'lists', + 'self', + 'qr', + 'web', + 'w:b', + 'recipient', + 'w:stats', + 'forbidden', + 'max_list_recipients', + 'en-AU', + 'en-GB', + 'es-MX', + 'pt-PT', + 'zh-Hans', + 'zh-Hant', + 'relayelection', + 'relaylatency', + 'interruption', + 'Bell.caf', + 'Boing.caf', + 'Glass.caf', + 'Harp.caf', + 'TimePassing.caf', + 'Tri-tone.caf', + 'Xylophone.caf', + 'aurora.m4r', + 'bamboo.m4r', + 'chord.m4r', + 'circles.m4r', + 'complete.m4r', + 'hello.m4r', + 'input.m4r', + 'keys.m4r', + 'note.m4r', + 'popcorn.m4r', + 'pulse.m4r', + 'synth.m4r', + 'Apex.m4r', + 'Beacon.m4r', + 'Bulletin.m4r', + 'By The Seaside.m4r', + 'Chimes.m4r', + 'Circuit.m4r', + 'Constellation.m4r', + 'Cosmic.m4r', + 'Crystals.m4r', + 'Hillside.m4r', + 'Illuminate.m4r', + 'Night Owl.m4r', + 'Opening.m4r', + 'Playtime.m4r', + 'Presto.m4r', + 'Radar.m4r', + 'Radiate.m4r', + 'Ripples.m4r', + 'Sencha.m4r', + 'Signal.m4r', + 'Silk.m4r', + 'Slow Rise.m4r', + 'Stargaze.m4r', + 'Summit.m4r', + 'Twinkle.m4r', + 'Uplift.m4r', + 'Waves.m4r', + 'eligible', + 'planned', + 'current', + 'future', + 'disable', + 'expire', + 'start', + 'stop', + 'accuracy', + 'speed', + 'bearing', + 'recording', + 'key', + 'identity', + 'w:gp2', + 'admin', + 'locked', + 'unlocked', + 'new', + 'battery', + 'archive', + 'adm', + 'plaintext_size', + 'plaintext_disabled', + 'plaintext_reenable_threshold', + 'compressed_size', + 'delivered', + 'everyone', + 'transport', + 'mspes', + 'e2e_groups', + 'e2e_images', + 'encr_media', + 'encrypt_v2', + 'encrypt_image', + 'encrypt_sends_push', + 'force_long_connect', + 'audio_opus', + 'video_max_edge', + 'call-id', + 'call', + 'preaccept', + 'accept', + 'offer', + 'reject', + 'busy', + 'te', + 'terminate', + 'begin', + 'end', + 'opus', + 'rtt', + 'token', + 'priority', + 'p2p', + 'rate', + 'amr', + 'ptt', + 'srtp', + 'os', + 'browser', + 'encrypt_group_gen2' ] @@ -487,4 +526,3 @@ def getIndex(self, token): return (self.secondaryDictionary.index(token), True) return None - From c2f83a8669057809b9d904b5f789354911ec5dbf Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 14:53:49 +0100 Subject: [PATCH 10/68] Updated decoder funcs --- yowsup/layers/coder/decoder.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 52fa4521d..fe4113748 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -73,14 +73,14 @@ def readPacked8(self, n, data): size = size & 0x7F text = bytearray(self.readArray(size, data)) data = binascii.hexlify(text) - out = "" + out = [] for i in range(0, len(data)): char = chr(data[i]) if type(data[i]) is int else data[i] #python2/3 compat val = ord(binascii.unhexlify("0%s" % char)) if i == size - 1 and val > 11 and n != 251: continue - out += chr(self.unpackByte(n, val)) + out.append(self.unpackByte(n, val)) - return out[0:len(out) - remove] + return out[0: -remove] def unpackByte(self, n, n2): if n == 251: @@ -194,7 +194,7 @@ def readString(self,token, data): raise Exception("readString couldn't reconstruct jid") if token in (251, 255): - return self.readPacked8(token, data) + return "".join(map(chr, self.readPacked8(token, data))) if token == 252: @@ -223,7 +223,7 @@ def readArray(self, length, data): return out def nextTreeInternal(self, data): - size = self.readListSize(self.readInt8(data)) + size = self.readListSize(self.readInt8(data), data) token = self.readInt8(data) if token == 1: token = self.readInt8(data) @@ -256,7 +256,7 @@ def nextTreeInternal(self, data): size = self.readInt31(data) nodeData = self.readArray(size, data) if read2 in (255, 251): - nodeData = self.readPacked8(read2, data) + nodeData = "".join(map(chr, self.readPacked8(read2, data))) nodeData = nodeData or self.readString(read2, data) return ProtocolTreeNode(tag, attribs, None, nodeData) From 7f2eab138456c6a1db3c093454d71b1e742fdccb Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 14:54:04 +0100 Subject: [PATCH 11/68] Encoder initial commit for wa1.6 t --- yowsup/layers/coder/encoder.py | 109 ++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/yowsup/layers/coder/encoder.py b/yowsup/layers/coder/encoder.py index 70c067194..c5209983b 100644 --- a/yowsup/layers/coder/encoder.py +++ b/yowsup/layers/coder/encoder.py @@ -13,7 +13,7 @@ def getStreamStartBytes(self, domain, resource): data.append(87) data.append(65) data.append(1) - data.append(5) + data.append(6) streamOpenAttributes = {"to": domain, "resource": resource} self.writeListStart(len(streamOpenAttributes) * 2 + 1, data) @@ -56,21 +56,30 @@ def writeAttributes(self, attributes, data): self.writeString(value, data); - def writeBytes(self, bytes, data): - - length = len(bytes) - if length >= 256: + def writeBytes(self, bytes_, data, packed = False): + size = len(bytes_) + toWrite = bytes_ + if size >= 0x100000: + data.append(254) + self.writeInt31(size, data) + elif size >= 0x100: data.append(253) - self.writeInt24(length, data) + self.writeInt20(size, data) else: - data.append(252) - self.writeInt8(length, data) - - for b in bytes: - if type(b) is int: - data.append(b) + r = None + if packed: + if size < 128: + r = self.tryPackAndWriteHeader(255, bytes_, data) + if r is None: + r = self.tryPackAndWriteHeader(251, bytes_, data) + + if r is None: + data.append(252) + self.writeInt8(size, data) else: - data.append(ord(b)) + toWrite = r + + data.extend(toWrite) def writeInt8(self, v, data): data.append(v & 0xFF) @@ -80,12 +89,21 @@ def writeInt16(self, v, data): data.append((v & 0xFF00) >> 8); data.append((v & 0xFF) >> 0); + def writeInt20(self, v, data): + data.append((0xF0000 & v) >> 16) + data.append((0xFF00 & v) >> 8) + data.append((v & 0xFF) >> 0) def writeInt24(self, v, data): data.append((v & 0xFF0000) >> 16) data.append((v & 0xFF00) >> 8) data.append((v & 0xFF) >> 0) + def writeInt31(self, v, data): + data.append((0x7F000000 & v) >> 24) + data.append((0xFF0000 & v) >> 16) + data.append((0xFF00 & v) >> 8) + data.append((v & 0xFF) >> 0) def writeListStart(self, i, data): if i == 0: @@ -97,14 +115,14 @@ def writeListStart(self, i, data): data.append(249) self.writeInt16(i, data) - def writeToken(self, intValue, data): - if intValue < 245: - data.append(intValue) - elif intValue <=500: - data.append(254) - data.append(intValue - 245) + def writeToken(self, token, data): + if token <= 255 and token >=0: + data.append(token) + else: + raise ValueError("Invalid token: %s" % token) + - def writeString(self, tag, data): + def writeString(self, tag, data, packed = False): tok = self.tokenDictionary.getIndex(tag) if tok: index, secondary = tok @@ -123,7 +141,7 @@ def writeString(self, tag, data): user = tag[0:atIndex] self.writeJid(user, server, data) except ValueError: - self.writeBytes(self.encodeString(tag), data) + self.writeBytes(self.encodeString(tag), data, packed) def encodeString(self, string): res = [] @@ -143,3 +161,52 @@ def writeJid(self, user, server, data): else: self.writeToken(0, data) self.writeString(server, data) + + + def tryPackAndWriteHeader(self, v, headerData, data): + size = len(headerData) + if size >= 128: + return None + + arr = [0] * int((size + 1) / 2) + for i in range(0, size): + packByte = self.packByte(v, headerData[i]) + if packByte == -1: + arr = [] + break + n2 = int(i / 2) + arr[n2] |= (packByte << 4 * (1 - i % 2)) + + if len(arr) > 0: + if size % 2 == 1: + arr[-1] |= 15 #0xF + data.append(v) + self.writeInt8(size %2 << 7 | len(arr), data) + return arr + + return None + + + + def packByte(self, v, n2): + if v == 251: + return self.packHex(n2) + if v == 255: + return self.packNibble(n2) + return -1 + + def packHex(self, n): + if n in range(48, 58): + return n - 48 + if n in range(65, 71): + return 10 + (n - 65) + return -1 + + def packNibble(self, n): + if n in (45, 46): + return 10 + (n - 45) + + if n in range(48, 58): + return n - 48 + + return -1 From bf64851fc241c16e2f0b8db169a8c26d493ea64f Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 15:02:54 +0100 Subject: [PATCH 12/68] Fixed read node data --- yowsup/layers/coder/decoder.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index fe4113748..07bea24d2 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -163,14 +163,12 @@ def readListSize(self,token, data): def readAttributes(self, attribCount, data): attribs = {} - for i in range(0, int(attribCount)): - key = self.readString(data.pop(0), data) - value = self.readString(data.pop(0), data) + key = self.readString(self.readInt8(data), data) + value = self.readString(self.readInt8(data), data) attribs[key]=value return attribs - def readString(self,token, data): if token == -1: raise Exception("-1 token in readString") @@ -196,7 +194,6 @@ def readString(self,token, data): if token in (251, 255): return "".join(map(chr, self.readPacked8(token, data))) - if token == 252: size8 = self.readInt8(data) buf8 = self.readArray(size8, data) @@ -256,9 +253,13 @@ def nextTreeInternal(self, data): size = self.readInt31(data) nodeData = self.readArray(size, data) if read2 in (255, 251): - nodeData = "".join(map(chr, self.readPacked8(read2, data))) + nodeData = self.readPacked8(read2, data) + + if nodeData: + nodeData = "".join(map(chr, nodeData)) + else: + nodeData = self.readString(read2, data) - nodeData = nodeData or self.readString(read2, data) return ProtocolTreeNode(tag, attribs, None, nodeData) def readList(self,token, data): From a0d61a3ca9b19ec2af8dc9005f1b52fa5faacb61 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 15:16:26 +0100 Subject: [PATCH 13/68] Fixed readPacked8 error --- yowsup/layers/coder/decoder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 07bea24d2..546956635 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -72,15 +72,18 @@ def readPacked8(self, n, data): remove = 1 size = size & 0x7F text = bytearray(self.readArray(size, data)) - data = binascii.hexlify(text) + hexData = binascii.hexlify(text) out = [] - for i in range(0, len(data)): - char = chr(data[i]) if type(data[i]) is int else data[i] #python2/3 compat + for i in range(0, len(hexData)): + char = chr(hexData[i]) if type(hexData[i]) is int else hexData[i] #python2/3 compat val = ord(binascii.unhexlify("0%s" % char)) - if i == size - 1 and val > 11 and n != 251: continue + if i == (size - 1) and val > 11 and n != 251: continue out.append(self.unpackByte(n, val)) - return out[0: -remove] + if remove: + out = out[0: -remove] + + return out def unpackByte(self, n, n2): if n == 251: From 52bc2d0f788b7d564dd601cd5f5d5b74aa13fba0 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 19:01:05 +0100 Subject: [PATCH 14/68] Fixed encoder --- yowsup/layers/coder/encoder.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/yowsup/layers/coder/encoder.py b/yowsup/layers/coder/encoder.py index c5209983b..3510e6a5e 100644 --- a/yowsup/layers/coder/encoder.py +++ b/yowsup/layers/coder/encoder.py @@ -37,9 +37,11 @@ def writeInternal(self, node, data): self.writeListStart(x, data) + self.writeString(node.tag, data) self.writeAttributes(node.attributes, data); + if node.data is not None: self.writeBytes(node.data, data) @@ -53,12 +55,20 @@ def writeAttributes(self, attributes, data): if attributes is not None: for key, value in attributes.items(): self.writeString(key, data); - self.writeString(value, data); + self.writeString(value, data, True); def writeBytes(self, bytes_, data, packed = False): - size = len(bytes_) - toWrite = bytes_ + bytes__ = [] + for b in bytes_: + if type(b) is int: + bytes__.append(b) + else: + bytes__.append(ord(b)) + + + size = len(bytes__) + toWrite = bytes__ if size >= 0x100000: data.append(254) self.writeInt31(size, data) @@ -69,9 +79,9 @@ def writeBytes(self, bytes_, data, packed = False): r = None if packed: if size < 128: - r = self.tryPackAndWriteHeader(255, bytes_, data) + r = self.tryPackAndWriteHeader(255, bytes__, data) if r is None: - r = self.tryPackAndWriteHeader(251, bytes_, data) + r = self.tryPackAndWriteHeader(251, bytes__, data) if r is None: data.append(252) @@ -157,7 +167,7 @@ def encodeString(self, string): def writeJid(self, user, server, data): data.append(250) if user is not None: - self.writeString(user, data) + self.writeString(user, data, True) else: self.writeToken(0, data) self.writeString(server, data) @@ -176,7 +186,6 @@ def tryPackAndWriteHeader(self, v, headerData, data): break n2 = int(i / 2) arr[n2] |= (packByte << 4 * (1 - i % 2)) - if len(arr) > 0: if size % 2 == 1: arr[-1] |= 15 #0xF From 8dc1d1bc9b4856d152b44ed3c584ab137abed078 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 21:42:36 +0100 Subject: [PATCH 15/68] Updated env Added manufacturer and build version --- yowsup/env/env.py | 7 +++++++ yowsup/env/env_android.py | 10 +++++++++- yowsup/env/env_s40.py | 3 +++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/yowsup/env/env.py b/yowsup/env/env.py index 7c7c1a48b..230a60881 100644 --- a/yowsup/env/env.py +++ b/yowsup/env/env.py @@ -69,10 +69,17 @@ def getOSName(self): def getDeviceName(self): pass + @abc.abstractmethod + def getManufacturer(self): + pass + @abc.abstractmethod def isAxolotlEnabled(self): pass + def getBuildVersion(self): + return "" + def getResource(self): return self.getOSName() + "-" + self.getVersion() diff --git a/yowsup/env/env_android.py b/yowsup/env/env_android.py index fdcaea625..6771b58e0 100644 --- a/yowsup/env/env_android.py +++ b/yowsup/env/env_android.py @@ -23,7 +23,9 @@ class AndroidYowsupEnv(YowsupEnv): _VERSION = "2.12.556" _OS_NAME = "Android" _OS_VERSION = "4.3" - _DEVICE_NAME = "GalaxyS3" + _DEVICE_NAME = "armani" + _MANUFACTURER = "Xiaomi" + _BUILD_VERSION = "JLS36C" _AXOLOTL = True def getVersion(self): @@ -38,6 +40,12 @@ def getOSVersion(self): def getDeviceName(self): return self.__class__._DEVICE_NAME + def getBuildVersion(self): + return self.__class__._BUILD_VERSION + + def getManufacturer(self): + return self.__class__._MANUFACTURER + def isAxolotlEnabled(self): return self.__class__._AXOLOTL diff --git a/yowsup/env/env_s40.py b/yowsup/env/env_s40.py index 3d2b85735..e92e8714b 100644 --- a/yowsup/env/env_s40.py +++ b/yowsup/env/env_s40.py @@ -22,6 +22,9 @@ def getOSVersion(self): def getDeviceName(self): return self.__class__._DEVICE_NAME + def getManufacturer(self): + return self.__class__._MANUFACTURER + def isAxolotlEnabled(self): return self.__class__._AXOLOTL From 7cd0c4d0db574d63475491274504db07339c2ae2 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 21:43:32 +0100 Subject: [PATCH 16/68] Updated auth for wa1.6 --- yowsup/layers/auth/layer_authentication.py | 28 +++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/yowsup/layers/auth/layer_authentication.py b/yowsup/layers/auth/layer_authentication.py index bd73cd516..2080070e8 100644 --- a/yowsup/layers/auth/layer_authentication.py +++ b/yowsup/layers/auth/layer_authentication.py @@ -8,8 +8,9 @@ from .protocolentities import * from yowsup.common.tools import StorageTools from .layer_interface_authentication import YowAuthenticationProtocolLayerInterface -from .protocolentities import StreamErrorProtocolEntity +from yowsup.env import CURRENT_ENV import base64 + class YowAuthenticationProtocolLayer(YowProtocolLayer): EVENT_LOGIN = "org.openwhatsapp.yowsup.event.auth.login" EVENT_AUTHED = "org.openwhatsapp.yowsup.event.auth.authed" @@ -49,7 +50,7 @@ def getUsername(self, full = False): else: prop = self.getProp(YowAuthenticationProtocolLayer.PROP_CREDENTIALS) return prop[0] if prop else None - + @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) def onConnected(self, yowLayerEvent): self.login() @@ -83,17 +84,17 @@ def handleChallenge(self, node): self._sendResponse(nodeEntity.getNonce()) def handleStreamError(self, node): - nodeEntity = StreamErrorProtocolEntity.fromProtocolTreeNode(node) - errorType = nodeEntity.getErrorType() - - if not errorType: + if node.getChild("text"): + nodeEntity = StreamErrorConflictProtocolEntity.fromProtocolTreeNode(node) + elif node.getChild("ack"): + nodeEntity = StreamErrorAckProtocolEntity.fromProtocolTreeNode(node) + else: raise AuthError("Unhandled stream:error node:\n%s" % node) - self.toUpper(nodeEntity) ##senders def _sendFeatures(self): - self.entityToLower(StreamFeaturesProtocolEntity(["readreceipts", "groups_v2", "privacy", "presence"])) + self.entityToLower(StreamFeaturesProtocolEntity([])) def _sendAuth(self): passive = self.getProp(self.__class__.PROP_PASSIVE, False) @@ -137,6 +138,17 @@ def generateAuthBlob(self, nonce): username_bytes = list(map(ord, self.credentials[0])) nums.extend(username_bytes) nums.extend(nonce) + nums.extend([48, 48, 48, 48, 48, 48, 48, 48]) + + strCat = "\x00\x00\x00\x00\x00\x00\x00\x00" + strCat += CURRENT_ENV.getOSVersion() + "\x00" + strCat += CURRENT_ENV.getManufacturer() + "\x00" + strCat += CURRENT_ENV.getDeviceName() + "\x00" + strCat += CURRENT_ENV.getBuildVersion() + + + nums.extend(CURRENT_ENV.getOSVersion()) + utcNow = str(int(TimeTools.utcTimestamp())) From 0404bab0cd4f97662e2e93704e7e5cf37e6d30cf Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Jan 2016 23:22:50 +0100 Subject: [PATCH 17/68] Fixed user agent format --- yowsup/env/env.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yowsup/env/env.py b/yowsup/env/env.py index 230a60881..a3ae0c8b4 100644 --- a/yowsup/env/env.py +++ b/yowsup/env/env.py @@ -15,7 +15,7 @@ class YowsupEnv(with_metaclass(YowsupEnvType, object)): __ENVS = {} __CURR = None - _USERAGENT_STRING = "WhatsApp/{WHATSAPP_VERSION} {OS_NAME}/{OS_VERSION} Device/{DEVICE_NAME}" + _USERAGENT_STRING = "WhatsApp/{WHATSAPP_VERSION} {OS_NAME}/{OS_VERSION} Device/{MANUFACTURER}-{DEVICE_NAME}" @classmethod def registerEnv(cls, envCls): @@ -88,5 +88,6 @@ def getUserAgent(self): WHATSAPP_VERSION = self.getVersion(), OS_NAME = self.getOSName(), OS_VERSION = self.getOSVersion(), + MANUFACTURER = self.getManufacturer(), DEVICE_NAME = self.getDeviceName() ) From f6c335b32ad32232d5825f7ddadd03755fbaa9b8 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 12 Jan 2016 00:26:56 +0100 Subject: [PATCH 18/68] New registration parameters --- yowsup/registration/existsrequest.py | 12 ++++++++ yowsup/registration/regrequest.py | 44 ++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/yowsup/registration/existsrequest.py b/yowsup/registration/existsrequest.py index 78a446ae9..91aab6817 100644 --- a/yowsup/registration/existsrequest.py +++ b/yowsup/registration/existsrequest.py @@ -1,6 +1,8 @@ from yowsup.common.http.warequest import WARequest from yowsup.common.http.waresponseparser import JSONResponseParser from yowsup.env import YowsupEnv +import os + class WAExistsRequest(WARequest): def __init__(self,cc, p_in, idx): @@ -12,6 +14,16 @@ def __init__(self,cc, p_in, idx): self.addParam("lg", "en") self.addParam("lc", "GB") self.addParam("token", YowsupEnv.getCurrent().getToken(p_in)) + self.addParam("mistyped", '6') + self.addParam('network_radio_type', '1') + self.addParam('simnum', '1') + self.addParam('s', '') + self.addParam('copiedrc', '1') + self.addParam('hasinrc', '1') + self.addParam('rcmatch', '1') + self.addParam('pid', os.getpid()) + self.addParam('extexist', '1') + self.addParam('extstate', '1') self.url = "v.whatsapp.net/v2/exist" diff --git a/yowsup/registration/regrequest.py b/yowsup/registration/regrequest.py index ec5e90019..f65b0a14b 100644 --- a/yowsup/registration/regrequest.py +++ b/yowsup/registration/regrequest.py @@ -1,27 +1,29 @@ ''' Copyright (c) <2012> Tarek Galal -Permission is hereby granted, free of charge, to any person obtaining a copy of this -software and associated documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights to use, copy, modify, -merge, publish, distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to the following +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR -A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR +A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ''' from yowsup.common.http.warequest import WARequest from yowsup.common.http.waresponseparser import JSONResponseParser from yowsup.common.tools import StorageTools +import hashlib +import os class WARegRequest(WARequest): @@ -38,12 +40,28 @@ def __init__(self,cc, p_in, code): self.addParam("id", idx) self.addParam("code", code) + self.addParam("lc", "GB") + self.addParam("lg", "en") + + self.addParam("mistyped", '6') + # self.addParam('network_radio_type', '1') + self.addParam('simnum', '1') + self.addParam('s', '') + self.addParam('copiedrc', '1') + self.addParam('hasinrc', '1') + self.addParam('rcmatch', '1') + self.addParam('pid', os.getpid()) + self.addParam('rchash', hashlib.sha256(os.urandom(20)).hexdigest()) + self.addParam('anhash', hashlib.md5(os.urandom(20)).hexdigest()) + self.addParam('extexist', '1') + self.addParam('extstate', '1') + self.url = "v.whatsapp.net/v2/register" self.pvars = ["status", "login", "pw", "type", "expiration", "kind", "price", "cost", "currency", "price_expiration", "reason","retry_after"] self.setParser(JSONResponseParser()) - + def register(self): - return self.send() \ No newline at end of file + return self.send() From 3d909e84a743e784ecd67196cb706cbe010a42d9 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 12 Jan 2016 00:48:19 +0100 Subject: [PATCH 19/68] Fixed decoder mishandling children nodes --- yowsup/layers/coder/decoder.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 546956635..85a5f3ee2 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -244,26 +244,27 @@ def nextTreeInternal(self, data): read2 = self.readInt8(data) nodeData = None + nodeChildren = None if self.isListTag(read2): - nodeData = self.readList(read2, data) - if read2 == 252: + nodeChildren = self.readList(read2, data) + elif read2 == 252: size = self.readInt8(data) nodeData = self.readArray(size, data) - if read2 == 253: + elif read2 == 253: size = self.readInt20(data) nodeData = self.readArray(size, data) - if read2 == 254: + elif read2 == 254: size = self.readInt31(data) nodeData = self.readArray(size, data) - if read2 in (255, 251): + elif read2 in (255, 251): nodeData = self.readPacked8(read2, data) + else: + nodeData = self.readString(read2, data) if nodeData: nodeData = "".join(map(chr, nodeData)) - else: - nodeData = self.readString(read2, data) - return ProtocolTreeNode(tag, attribs, None, nodeData) + return ProtocolTreeNode(tag, attribs, nodeChildren, nodeData) def readList(self,token, data): size = self.readListSize(token, data) From 69b08a6ef3f3ba89cbf21216d5be616da00617ab Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 12 Jan 2016 00:57:41 +0100 Subject: [PATCH 20/68] Fixed authentication --- yowsup/layers/auth/layer_authentication.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/yowsup/layers/auth/layer_authentication.py b/yowsup/layers/auth/layer_authentication.py index 2080070e8..8962d73b4 100644 --- a/yowsup/layers/auth/layer_authentication.py +++ b/yowsup/layers/auth/layer_authentication.py @@ -138,23 +138,17 @@ def generateAuthBlob(self, nonce): username_bytes = list(map(ord, self.credentials[0])) nums.extend(username_bytes) nums.extend(nonce) - nums.extend([48, 48, 48, 48, 48, 48, 48, 48]) + + utcNow = str(int(TimeTools.utcTimestamp())) + time_bytes = list(map(ord, utcNow)) + nums.extend(time_bytes) strCat = "\x00\x00\x00\x00\x00\x00\x00\x00" strCat += CURRENT_ENV.getOSVersion() + "\x00" strCat += CURRENT_ENV.getManufacturer() + "\x00" strCat += CURRENT_ENV.getDeviceName() + "\x00" strCat += CURRENT_ENV.getBuildVersion() - - - nums.extend(CURRENT_ENV.getOSVersion()) - - - utcNow = str(int(TimeTools.utcTimestamp())) - - time_bytes = list(map(ord, utcNow)) - - nums.extend(time_bytes) + nums.extend(list(map(ord, strCat))) encoded = outputKey.encodeMessage(nums, 0, 4, len(nums) - 4) authBlob = "".join(map(chr, encoded)) From 1262d350e106d46ee4601de33f18d840b04a8988 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 12 Jan 2016 01:26:54 +0100 Subject: [PATCH 21/68] Fixed readPacked8 --- yowsup/layers/coder/decoder.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 85a5f3ee2..9d5e66bfd 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -68,20 +68,20 @@ def readNibble(self, data): def readPacked8(self, n, data): size = self.readInt8(data) remove = 0 - if (size & 0x80) != 0 and n == 251: + if (size & 0x80) != 0: remove = 1 - size = size & 0x7F + size = size & 0x7F text = bytearray(self.readArray(size, data)) hexData = binascii.hexlify(text) out = [] - for i in range(0, len(hexData)): - char = chr(hexData[i]) if type(hexData[i]) is int else hexData[i] #python2/3 compat - val = ord(binascii.unhexlify("0%s" % char)) - if i == (size - 1) and val > 11 and n != 251: continue - out.append(self.unpackByte(n, val)) - - if remove: - out = out[0: -remove] + if remove == 0: + for i in range(0, len(hexData)): + char = chr(hexData[i]) if type(hexData[i]) is int else hexData[i] #python2/3 compat + val = ord(binascii.unhexlify("0%s" % char)) + if i == (size - 1) and val > 11 and n != 251: continue + out.append(self.unpackByte(n, val)) + else: + out = hexData[0: -remove] return out From 052e3421028c9f908d061f287150435ccb6b48cd Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 12 Jan 2016 01:53:35 +0100 Subject: [PATCH 22/68] readPacked8 uppercases hexlified string, fixes wrong ids (in receipts) --- yowsup/layers/coder/decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 9d5e66bfd..e958a8e69 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -72,7 +72,7 @@ def readPacked8(self, n, data): remove = 1 size = size & 0x7F text = bytearray(self.readArray(size, data)) - hexData = binascii.hexlify(text) + hexData = binascii.hexlify(text).upper() out = [] if remove == 0: for i in range(0, len(hexData)): From d1b470a8afe7cbaec803ca6bec2bcb8949dea8cf Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 12 Jan 2016 21:35:52 +0100 Subject: [PATCH 23/68] Fixed getTokenDouble returning None --- yowsup/layers/coder/decoder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index e958a8e69..9bb00c35d 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -1,6 +1,7 @@ from yowsup.structs import ProtocolTreeNode import math import binascii +import sys class ReadDecoder: def __init__(self, tokenDictionary): self.streamStarted = False; @@ -26,11 +27,12 @@ def getToken(self, index, data): def getTokenDouble(self, n, n2): pos = n2 + n * 256 - ret = "" token = self.tokenDictionary.getToken(pos, True) if not token: raise ValueError("Invalid token %s" % pos) + return token + def streamStart(self, data): self.streamStarted = True tag = data.pop(0) @@ -81,7 +83,7 @@ def readPacked8(self, n, data): if i == (size - 1) and val > 11 and n != 251: continue out.append(self.unpackByte(n, val)) else: - out = hexData[0: -remove] + out = map(ord, list(hexData[0: -remove])) if sys.version_info < (3,0) else list(hexData[0: -remove]) return out From c095b16a8009f346b6d90c6365de773d3b4ac3d7 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Jan 2016 10:09:24 +0100 Subject: [PATCH 24/68] Fixed readint20 --- yowsup/layers/coder/decoder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 9bb00c35d..afd6bd296 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -133,9 +133,9 @@ def readInt16(self, data): return "" def readInt20(self, data): - int1 = self.data.pop(0) - int2 = self.data.pop(1) - int3 = self.data.pop(2) + int1 = data.pop(0) + int2 = data.pop(0) + int3 = data.pop(0) return ((int1 & 0xF) << 16) + (int2 << 8) + (int3 << 0) def readInt24(self, data): From 2aa0ad1abc058548517eb48598c1f0d5da088c03 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Jan 2016 10:14:31 +0100 Subject: [PATCH 25/68] Added Message::getAuthor Gets participant when group, and sender when non group message --- .../protocol_messages/protocolentities/message.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/yowsup/layers/protocol_messages/protocolentities/message.py b/yowsup/layers/protocol_messages/protocolentities/message.py index 98e9f71c5..615a41ad8 100644 --- a/yowsup/layers/protocol_messages/protocolentities/message.py +++ b/yowsup/layers/protocol_messages/protocolentities/message.py @@ -7,7 +7,7 @@ class MessageProtocolEntity(ProtocolEntity): MESSAGE_TYPE_TEXT = "text" MESSAGE_TYPE_MEDIA = "media" - def __init__(self, _type, _id = None, _from = None, to = None, notify = None, timestamp = None, + def __init__(self, _type, _id = None, _from = None, to = None, notify = None, timestamp = None, participant = None, offline = None, retry = None): assert (to or _from), "Must specify either to or _from jids to create the message" @@ -46,9 +46,12 @@ def getTo(self, full = True): def getParticipant(self, full = True): return self.participant if full else self.participant.split('@')[0] + def getAuthor(self, full = True): + return self.getParticipant(full) if self.isGroupMessage() else self.getFrom(full) + def getNotify(self): return self.notify - + def toProtocolTreeNode(self): attribs = { "type" : self._type, @@ -91,7 +94,7 @@ def isGroupMessage(self): def __str__(self): out = "Message:\n" out += "ID: %s\n" % self._id - out += "To: %s\n" % self.to if self.isOutgoing() else "From: %s\n" % self._from + out += "To: %s\n" % self.to if self.isOutgoing() else "From: %s\n" % self._from out += "Type: %s\n" % self._type out += "Timestamp: %s\n" % self.timestamp if self.participant: @@ -112,7 +115,7 @@ def forward(self, to, _id = None): def fromProtocolTreeNode(node): return MessageProtocolEntity( - node.getAttributeValue("type"), + node.getAttributeValue("type"), node.getAttributeValue("id"), node.getAttributeValue("from"), node.getAttributeValue("to"), From b6a0953de5a5d5edf27edca7d1c82bdfe56081ae Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Jan 2016 10:19:34 +0100 Subject: [PATCH 26/68] Added SenderKey store --- .../axolotl/store/sqlite/liteaxolotlstore.py | 10 ++++- .../store/sqlite/litesenderkeystore.py | 43 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py diff --git a/yowsup/layers/axolotl/store/sqlite/liteaxolotlstore.py b/yowsup/layers/axolotl/store/sqlite/liteaxolotlstore.py index 2dadec0b3..b79d399c1 100644 --- a/yowsup/layers/axolotl/store/sqlite/liteaxolotlstore.py +++ b/yowsup/layers/axolotl/store/sqlite/liteaxolotlstore.py @@ -3,6 +3,7 @@ from .liteprekeystore import LitePreKeyStore from .litesessionstore import LiteSessionStore from .litesignedprekeystore import LiteSignedPreKeyStore +from .litesenderkeystore import LiteSenderKeyStore import sqlite3 class LiteAxolotlStore(AxolotlStore): def __init__(self, db): @@ -12,6 +13,7 @@ def __init__(self, db): self.preKeyStore = LitePreKeyStore(conn) self.signedPreKeyStore = LiteSignedPreKeyStore(conn) self.sessionStore = LiteSessionStore(conn) + self.senderKeyStore = LiteSenderKeyStore(conn) def getIdentityKeyPair(self): return self.identityKeyStore.getIdentityKeyPair() @@ -74,4 +76,10 @@ def containsSignedPreKey(self, signedPreKeyId): return self.signedPreKeyStore.containsSignedPreKey(signedPreKeyId) def removeSignedPreKey(self, signedPreKeyId): - self.signedPreKeyStore.removeSignedPreKey(signedPreKeyId) \ No newline at end of file + self.signedPreKeyStore.removeSignedPreKey(signedPreKeyId) + + def loadSenderKey(self, senderKeyName): + return self.senderKeyStore.loadSenderKey(senderKeyName) + + def storeSenderKey(self, senderKeyName, senderKeyRecord): + self.senderKeyStore.storeSenderKey(senderKeyName, senderKeyRecord) diff --git a/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py b/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py new file mode 100644 index 000000000..751166a69 --- /dev/null +++ b/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py @@ -0,0 +1,43 @@ +from axolotl.groups.state.senderkeystore import SenderKeyStore +from axolotl.groups.state.senderkeyrecord import SenderKeyRecord +import sqlite3 +class LiteSenderKeyStore(SenderKeyStore): + def __init__(self, dbConn): + """ + :type dbConn: Connection + """ + self.dbConn = dbConn + dbConn.execute("CREATE TABLE IF NOT EXISTS sender_keys (_id INTEGER PRIMARY KEY AUTOINCREMENT," + "group_id TEXT NOT NULL," + "sender_id INTEGER NOT NULL, record BLOB);") + + dbConn.execute("CREATE UNIQUE INDEX IF NOT EXISTS sender_keys_idx ON sender_keys (group_id, sender_id);") + + def storeSenderKey(self, senderKeyName, senderKeyRecord): + """ + :type senderKeyName: SenderKeName + :type senderKeyRecord: SenderKeyRecord + """ + q = "INSERT INTO sender_keys (group_id, sender_id, record) VALUES(?,?, ?)" + cursor = self.dbConn.cursor() + try: + cursor.execute(q, (senderKeyName.getGroupId(), senderKeyName.getSender().getName(), senderKeyRecord.serialize())) + self.dbConn.commit() + except sqlite3.IntegrityError as e: + q = "UPDATE sender_keys set record = ? WHERE group_id = ? and sender_id = ?" + cursor = self.dbConn.cursor() + cursor.execute(q, (senderKeyRecord.serialize(), senderKeyName.getGroupId(), senderKeyName.getSender().getName())) + self.dbConn.commit() + + def loadSenderKey(self, senderKeyName): + """ + :type senderKeyName: SenderKeyName + """ + q = "SELECT record FROM sender_keys WHERE group_id = ? and sender_id = ?" + cursor = self.dbConn.cursor() + cursor.execute(q, (senderKeyName.getGroupId(), senderKeyName.getSender().getName())) + + result = cursor.fetchone() + if not result: + return SenderKeyRecord() + return SenderKeyRecord(serialized = result[0]) From 534f2446c5e03cdf0cc7a3b91f5ec62fb14a20c9 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Jan 2016 10:21:51 +0100 Subject: [PATCH 27/68] Supported multiple child enc nodes --- yowsup/layers/axolotl/protocolentities/enc.py | 29 +++++++++++++ .../protocolentities/message_encrypted.py | 42 +++++++------------ 2 files changed, 45 insertions(+), 26 deletions(-) create mode 100644 yowsup/layers/axolotl/protocolentities/enc.py diff --git a/yowsup/layers/axolotl/protocolentities/enc.py b/yowsup/layers/axolotl/protocolentities/enc.py new file mode 100644 index 000000000..5aea2608c --- /dev/null +++ b/yowsup/layers/axolotl/protocolentities/enc.py @@ -0,0 +1,29 @@ +from yowsup.structs import ProtocolEntity, ProtocolTreeNode + +class EncProtocolEntity(ProtocolEntity): + TYPE_PKMSG = "pkmsg" + TYPE_MSG = "msg" + TYPE_SKMSG = "skmsg" + TYPES = (TYPE_PKMSG, TYPE_MSG, TYPE_SKMSG) + def __init__(self, type, version, data): + assert type in self.__class__.TYPES, "Unknown message enc type %s" % type + super(EncProtocolEntity, self).__init__("enc") + self.type = type + self.version = int(version) + self.data = data + + def getType(self): + return self.type + + def getVersion(self): + return self.version + + def getData(self): + return self.data + + def toProtocolTreeNode(self): + return ProtocolTreeNode("enc", {"type": self.type, "v": str(self.version)}, data = self.data) + + @staticmethod + def fromProtocolTreeNode(node): + return EncProtocolEntity(node["type"], node["v"], node.data) diff --git a/yowsup/layers/axolotl/protocolentities/message_encrypted.py b/yowsup/layers/axolotl/protocolentities/message_encrypted.py index c01439ce9..cfe9d2c7b 100644 --- a/yowsup/layers/axolotl/protocolentities/message_encrypted.py +++ b/yowsup/layers/axolotl/protocolentities/message_encrypted.py @@ -1,6 +1,7 @@ from yowsup.layers.protocol_messages.protocolentities import MessageProtocolEntity from yowsup.structs import ProtocolTreeNode import sys +from yowsup.layers.axolotl.protocolentities.enc import EncProtocolEntity class EncryptedMessageProtocolEntity(MessageProtocolEntity): ''' @@ -9,46 +10,35 @@ class EncryptedMessageProtocolEntity(MessageProtocolEntity): ''' - TYPE_PKMSG = "pkmsg" - TYPE_MSG = "msg" - - def __init__(self, encType, encVersion, encData, _type, _id = None, _from = None, to = None, notify = None, timestamp = None, + def __init__(self, encEntities, _type, _id = None, _from = None, to = None, notify = None, timestamp = None, participant = None, offline = None, retry = None ): super(EncryptedMessageProtocolEntity, self).__init__(_type, _id = _id, _from = _from, to = to, notify = notify, timestamp = timestamp, participant = participant, offline = offline, retry = retry) + self.setEncEntities(encEntities) - self.setEncProps(encType, encVersion, encData) - - def setEncProps(self, encType, encVersion, encData): - assert encType in "pkmsg", "msg" - self.encType = encType - self.encVersion = int(encVersion) - self.encData = encData - - def getEncType(self): - return self.encType + def setEncEntities(self, encEntities): + self.encEntities = {} + assert len(encEntities), "Must have at least 1 enc entity" + for enc in encEntities: + self.encEntities[enc.type] = enc - def getEncData(self): - return self.encData + def getEnc(self, encType): + if encType in self.encEntities: + return self.encEntities[encType] - def getVersion(self): - return self.encVersion + return None def toProtocolTreeNode(self): node = super(EncryptedMessageProtocolEntity, self).toProtocolTreeNode() - encNode = ProtocolTreeNode("enc", data = self.encData) - encNode["type"] = self.encType - encNode["v"] = str(self.encVersion) + for key, enc in self.encEntities.items(): + node.addChild(enc.toProtocolTreeNode()) - node.addChild(encNode) return node @staticmethod def fromProtocolTreeNode(node): entity = MessageProtocolEntity.fromProtocolTreeNode(node) entity.__class__ = EncryptedMessageProtocolEntity - encNode = node.getChild("enc") - entity.setEncProps(encNode["type"], encNode["v"], - encNode.data.encode('latin-1') if sys.version_info >= (3,0) else encNode.data) - return entity \ No newline at end of file + entity.setEncEntities([EncProtocolEntity.fromProtocolTreeNode(encNode) for encNode in node.getAllChildren("enc")]) + return entity From 1e1affa72b7229b5bf4f39d7be6dd112c9f7396b Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Jan 2016 10:37:38 +0100 Subject: [PATCH 28/68] Added e2e protobufs --- yowsup/layers/axolotl/e2e_pb2.py | 532 ++++++++++++++++++++++++++ yowsup/layers/axolotl/proto/e2e.proto | 62 +++ 2 files changed, 594 insertions(+) create mode 100644 yowsup/layers/axolotl/e2e_pb2.py create mode 100644 yowsup/layers/axolotl/proto/e2e.proto diff --git a/yowsup/layers/axolotl/e2e_pb2.py b/yowsup/layers/axolotl/e2e_pb2.py new file mode 100644 index 000000000..96a45e941 --- /dev/null +++ b/yowsup/layers/axolotl/e2e_pb2.py @@ -0,0 +1,532 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: e2e.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='e2e.proto', + package='', + serialized_pb=_b('\n\te2e.proto\"\xa9\x02\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12\x43\n\x1csenderKeyDistributionMessage\x18\x02 \x01(\x0b\x32\x1d.SenderKeyDistributionMessage\x12#\n\x0cimageMessage\x18\x03 \x01(\x0b\x32\r.ImageMessage\x12\'\n\x0e\x63ontactMessage\x18\x04 \x01(\x0b\x32\x0f.ContactMessage\x12)\n\x0flocationMessage\x18\x05 \x01(\x0b\x32\x10.LocationMessage\x12)\n\x0f\x64ocumentMessage\x18\x07 \x01(\x0b\x32\x10.DocumentMessage\x12\x1f\n\nurlMessage\x18\x06 \x01(\x0b\x32\x0b.UrlMessage\"\\\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x01(\x0c\x12+\n#axolotlSenderKeyDistributionMessage\x18\x02 \x01(\x0c\"\xae\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x01(\x0c\x12\x10\n\x08mimeType\x18\x02 \x01(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x01(\t\x12\x12\n\nfileSha256\x18\x04 \x01(\x0c\x12\x12\n\nfileLength\x18\x05 \x01(\x04\x12\x0e\n\x06height\x18\x06 \x01(\r\x12\r\n\x05width\x18\x07 \x01(\r\x12\x10\n\x08mediaKey\x18\x08 \x01(\x0c\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\x0c\"\x87\x01\n\x0fLocationMessage\x12\x17\n\x0f\x64\x65greesLatitude\x18\x01 \x01(\x01\x12\x18\n\x10\x64\x65greesLongitude\x18\x02 \x01(\x01\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\x0c\"\xa3\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x10\n\x08mimeType\x18\x02 \x01(\t\x12\r\n\x05title\x18\x03 \x01(\t\x12\x12\n\nfileSha256\x18\x04 \x01(\x0c\x12\x12\n\nfileLength\x18\x05 \x01(\x04\x12\x11\n\tpageCount\x18\x06 \x01(\r\x12\x10\n\x08mediaKey\x18\x07 \x01(\x0c\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\x0c\"\x80\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x01(\t\x12\x13\n\x0bmatchedText\x18\x02 \x01(\t\x12\x14\n\x0c\x63\x61nonicalUrl\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05title\x18\x06 \x01(\t\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\t\"4\n\x0e\x43ontactMessage\x12\x13\n\x0b\x64isplayName\x18\x01 \x01(\t\x12\r\n\x05vcard\x18\x10 \x01(\t') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_MESSAGE = _descriptor.Descriptor( + name='Message', + full_name='Message', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='conversation', full_name='Message.conversation', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='senderKeyDistributionMessage', full_name='Message.senderKeyDistributionMessage', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='imageMessage', full_name='Message.imageMessage', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='contactMessage', full_name='Message.contactMessage', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='locationMessage', full_name='Message.locationMessage', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='documentMessage', full_name='Message.documentMessage', index=5, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='urlMessage', full_name='Message.urlMessage', index=6, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=14, + serialized_end=311, +) + + +_SENDERKEYDISTRIBUTIONMESSAGE = _descriptor.Descriptor( + name='SenderKeyDistributionMessage', + full_name='SenderKeyDistributionMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='groupId', full_name='SenderKeyDistributionMessage.groupId', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='axolotlSenderKeyDistributionMessage', full_name='SenderKeyDistributionMessage.axolotlSenderKeyDistributionMessage', index=1, + number=2, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=313, + serialized_end=405, +) + + +_IMAGEMESSAGE = _descriptor.Descriptor( + name='ImageMessage', + full_name='ImageMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='ImageMessage.url', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mimeType', full_name='ImageMessage.mimeType', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='caption', full_name='ImageMessage.caption', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fileSha256', full_name='ImageMessage.fileSha256', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fileLength', full_name='ImageMessage.fileLength', index=4, + number=5, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='height', full_name='ImageMessage.height', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='width', full_name='ImageMessage.width', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mediaKey', full_name='ImageMessage.mediaKey', index=7, + number=8, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='jpegThumbnail', full_name='ImageMessage.jpegThumbnail', index=8, + number=16, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=408, + serialized_end=582, +) + + +_LOCATIONMESSAGE = _descriptor.Descriptor( + name='LocationMessage', + full_name='LocationMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='degreesLatitude', full_name='LocationMessage.degreesLatitude', index=0, + number=1, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='degreesLongitude', full_name='LocationMessage.degreesLongitude', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='name', full_name='LocationMessage.name', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='address', full_name='LocationMessage.address', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='url', full_name='LocationMessage.url', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='jpegThumbnail', full_name='LocationMessage.jpegThumbnail', index=5, + number=16, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=585, + serialized_end=720, +) + + +_DOCUMENTMESSAGE = _descriptor.Descriptor( + name='DocumentMessage', + full_name='DocumentMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='DocumentMessage.url', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mimeType', full_name='DocumentMessage.mimeType', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='title', full_name='DocumentMessage.title', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fileSha256', full_name='DocumentMessage.fileSha256', index=3, + number=4, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fileLength', full_name='DocumentMessage.fileLength', index=4, + number=5, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pageCount', full_name='DocumentMessage.pageCount', index=5, + number=6, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mediaKey', full_name='DocumentMessage.mediaKey', index=6, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='jpegThumbnail', full_name='DocumentMessage.jpegThumbnail', index=7, + number=16, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=723, + serialized_end=886, +) + + +_URLMESSAGE = _descriptor.Descriptor( + name='UrlMessage', + full_name='UrlMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='text', full_name='UrlMessage.text', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='matchedText', full_name='UrlMessage.matchedText', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='canonicalUrl', full_name='UrlMessage.canonicalUrl', index=2, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='description', full_name='UrlMessage.description', index=3, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='title', full_name='UrlMessage.title', index=4, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='jpegThumbnail', full_name='UrlMessage.jpegThumbnail', index=5, + number=16, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=889, + serialized_end=1017, +) + + +_CONTACTMESSAGE = _descriptor.Descriptor( + name='ContactMessage', + full_name='ContactMessage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='displayName', full_name='ContactMessage.displayName', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vcard', full_name='ContactMessage.vcard', index=1, + number=16, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + serialized_start=1019, + serialized_end=1071, +) + +_MESSAGE.fields_by_name['senderKeyDistributionMessage'].message_type = _SENDERKEYDISTRIBUTIONMESSAGE +_MESSAGE.fields_by_name['imageMessage'].message_type = _IMAGEMESSAGE +_MESSAGE.fields_by_name['contactMessage'].message_type = _CONTACTMESSAGE +_MESSAGE.fields_by_name['locationMessage'].message_type = _LOCATIONMESSAGE +_MESSAGE.fields_by_name['documentMessage'].message_type = _DOCUMENTMESSAGE +_MESSAGE.fields_by_name['urlMessage'].message_type = _URLMESSAGE +DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE +DESCRIPTOR.message_types_by_name['SenderKeyDistributionMessage'] = _SENDERKEYDISTRIBUTIONMESSAGE +DESCRIPTOR.message_types_by_name['ImageMessage'] = _IMAGEMESSAGE +DESCRIPTOR.message_types_by_name['LocationMessage'] = _LOCATIONMESSAGE +DESCRIPTOR.message_types_by_name['DocumentMessage'] = _DOCUMENTMESSAGE +DESCRIPTOR.message_types_by_name['UrlMessage'] = _URLMESSAGE +DESCRIPTOR.message_types_by_name['ContactMessage'] = _CONTACTMESSAGE + +Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( + DESCRIPTOR = _MESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:Message) + )) +_sym_db.RegisterMessage(Message) + +SenderKeyDistributionMessage = _reflection.GeneratedProtocolMessageType('SenderKeyDistributionMessage', (_message.Message,), dict( + DESCRIPTOR = _SENDERKEYDISTRIBUTIONMESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:SenderKeyDistributionMessage) + )) +_sym_db.RegisterMessage(SenderKeyDistributionMessage) + +ImageMessage = _reflection.GeneratedProtocolMessageType('ImageMessage', (_message.Message,), dict( + DESCRIPTOR = _IMAGEMESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:ImageMessage) + )) +_sym_db.RegisterMessage(ImageMessage) + +LocationMessage = _reflection.GeneratedProtocolMessageType('LocationMessage', (_message.Message,), dict( + DESCRIPTOR = _LOCATIONMESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:LocationMessage) + )) +_sym_db.RegisterMessage(LocationMessage) + +DocumentMessage = _reflection.GeneratedProtocolMessageType('DocumentMessage', (_message.Message,), dict( + DESCRIPTOR = _DOCUMENTMESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:DocumentMessage) + )) +_sym_db.RegisterMessage(DocumentMessage) + +UrlMessage = _reflection.GeneratedProtocolMessageType('UrlMessage', (_message.Message,), dict( + DESCRIPTOR = _URLMESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:UrlMessage) + )) +_sym_db.RegisterMessage(UrlMessage) + +ContactMessage = _reflection.GeneratedProtocolMessageType('ContactMessage', (_message.Message,), dict( + DESCRIPTOR = _CONTACTMESSAGE, + __module__ = 'e2e_pb2' + # @@protoc_insertion_point(class_scope:ContactMessage) + )) +_sym_db.RegisterMessage(ContactMessage) + + +# @@protoc_insertion_point(module_scope) diff --git a/yowsup/layers/axolotl/proto/e2e.proto b/yowsup/layers/axolotl/proto/e2e.proto new file mode 100644 index 000000000..c99508197 --- /dev/null +++ b/yowsup/layers/axolotl/proto/e2e.proto @@ -0,0 +1,62 @@ +message Message { + optional string conversation = 1; + optional SenderKeyDistributionMessage senderKeyDistributionMessage = 2; + optional ImageMessage imageMessage = 3; + optional ContactMessage contactMessage = 4; + optional LocationMessage locationMessage = 5; + optional DocumentMessage documentMessage = 7; + optional UrlMessage urlMessage = 6; +} + +message SenderKeyDistributionMessage { + optional bytes groupId = 1; + optional bytes axolotlSenderKeyDistributionMessage = 2; +} + +message ImageMessage { + optional bytes url = 1; + optional string mimeType = 2; + optional string caption = 3; + optional bytes fileSha256 = 4; + optional uint64 fileLength = 5; + optional uint32 height = 6; + optional uint32 width = 7; + optional bytes mediaKey = 8; + optional bytes jpegThumbnail = 16; +} + +message LocationMessage { + optional double degreesLatitude = 1; + optional double degreesLongitude = 2; + optional string name = 3; + optional string address = 4; + optional string url = 5; + optional bytes jpegThumbnail = 16; + +} +message DocumentMessage { + optional string url = 1; + optional string mimeType = 2; + optional string title = 3; + optional bytes fileSha256 = 4; + optional uint64 fileLength = 5; + optional uint32 pageCount = 6; + optional bytes mediaKey = 7; + optional bytes jpegThumbnail = 16; +} + + +message UrlMessage { + optional string text = 1; + optional string matchedText = 2; + optional string canonicalUrl = 4; + optional string description = 5; + optional string title = 6; + optional string jpegThumbnail = 16; +} + + +message ContactMessage { + optional string displayName = 1; + optional string vcard = 16; +} From 07638ce4e815235ebf4c618da469066352ed075f Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Jan 2016 11:43:29 +0200 Subject: [PATCH 29/68] Initial support for encrypted groups Handle skmsg Handle messages with multiple enc nodes Property decode v2 e2e messages using protobuf Fixed retry receipt not handling group participants --- yowsup/layers/axolotl/layer.py | 188 ++++++++++++------ .../axolotl/protocolentities/__init__.py | 4 +- .../receipt_outgoing_retry.py | 13 +- 3 files changed, 139 insertions(+), 66 deletions(-) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index 2e6f449a5..c21fcd037 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -1,34 +1,40 @@ from yowsup.layers import YowProtocolLayer, YowLayerEvent, EventCallback -from .protocolentities import SetKeysIqProtocolEntity -from axolotl.util.keyhelper import KeyHelper -from .store.sqlite.liteaxolotlstore import LiteAxolotlStore -from axolotl.sessionbuilder import SessionBuilder from yowsup.layers.protocol_messages.protocolentities.message import MessageProtocolEntity +from yowsup.layers.protocol_messages.protocolentities.message_text import TextMessageProtocolEntity from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity from yowsup.layers.network.layer import YowNetworkLayer from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer -from axolotl.ecc.curve import Curve +from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity +from yowsup.layers.axolotl.e2e_pb2 import Message +from yowsup.layers.axolotl.store.sqlite.liteaxolotlstore import LiteAxolotlStore +from yowsup.layers.axolotl.protocolentities import * +from yowsup.structs import ProtocolTreeNode from yowsup.common.tools import StorageTools + +from axolotl.sessionbuilder import SessionBuilder +from axolotl.util.keyhelper import KeyHelper +from axolotl.ecc.curve import Curve from axolotl.protocol.prekeywhispermessage import PreKeyWhisperMessage from axolotl.protocol.whispermessage import WhisperMessage -from .protocolentities import EncryptedMessageProtocolEntity +from axolotl.protocol.senderkeymessage import SenderKeyMessage from axolotl.sessioncipher import SessionCipher -from yowsup.structs import ProtocolTreeNode -from .protocolentities import GetKeysIqProtocolEntity, ResultGetKeysIqProtocolEntity +from axolotl.groups.groupcipher import GroupCipher from axolotl.util.hexutil import HexUtil from axolotl.invalidmessageexception import InvalidMessageException from axolotl.duplicatemessagexception import DuplicateMessageException -from .protocolentities import EncryptNotification -from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity from axolotl.invalidkeyidexception import InvalidKeyIdException from axolotl.nosessionexception import NoSessionException from axolotl.untrustedidentityexception import UntrustedIdentityException -from .protocolentities.receipt_outgoing_retry import RetryOutgoingReceiptProtocolEntity -from yowsup.common import YowConstants +from axolotl.axolotladdress import AxolotlAddress +from axolotl.groups.senderkeyname import SenderKeyName +from axolotl.groups.groupsessionbuilder import GroupSessionBuilder +from axolotl.protocol.senderkeydistributionmessage import SenderKeyDistributionMessage + import binascii +import copy import sys - import logging + logger = logging.getLogger(__name__) class YowAxolotlLayer(YowProtocolLayer): @@ -45,6 +51,7 @@ def __init__(self): self.state = self.__class__._STATE_INIT self.sessionCiphers = {} + self.groupCiphers = {} self.pendingMessages = {} self.pendingIncomingMessages = {} self.skipEncJids = [] @@ -86,18 +93,18 @@ def isGenKeysState(self): @EventCallback(EVENT_PREKEYS_SET) def onPreKeysSet(self, yowLayerEvent): self.sendKeys(fresh=False) - + @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) def onConnected(self, yowLayerEvent): if self.isInitState(): self.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, True) - + @EventCallback(YowAuthenticationProtocolLayer.EVENT_AUTHED) def onAuthed(self, yowLayerEvent): if yowLayerEvent.getArg("passive") and self.isInitState(): logger.info("Axolotl layer is generating keys") self.sendKeys() - + @EventCallback(YowNetworkLayer.EVENT_STATE_DISCONNECTED) def onDisconnected(self, yowLayerEvent): if self.isGenKeysState(): @@ -110,7 +117,7 @@ def onDisconnected(self, yowLayerEvent): self.store = None def send(self, node): - if node.tag == "message" and node["type"] == "text" and node["to"] not in self.skipEncJids and not YowConstants.WHATSAPP_GROUP_SERVER in node["to"]: + if node.tag == "message" and node["type"] == "text" and node["to"] not in self.skipEncJids: self.handlePlaintextNode(node) return self.toLower(node) @@ -196,9 +203,10 @@ def handlePlaintextNode(self, node): version = 1 ciphertext = sessionCipher.encrypt(plaintext) encEntity = EncryptedMessageProtocolEntity( - EncryptedMessageProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncryptedMessageProtocolEntity.TYPE_PKMSG , + [ + EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , version, - ciphertext.serialize(), + ciphertext.serialize())], MessageProtocolEntity.MESSAGE_TYPE_TEXT, _id= node["id"], to = node["to"], @@ -221,32 +229,35 @@ def encodeInt7bit(self, value): return out def handleEncMessage(self, node): + encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) + isGroup = node["from"] if node["participant"] is None else node["from"] + senderJid = node["participant"] if isGroup else node["from"] + encNode = None + if node.getChild("enc")["v"] == "2" and senderJid not in self.v2Jids: + self.v2Jids.append(senderJid) try: - if node.getChild("enc")["v"] == "2" and node["from"] not in self.v2Jids: - self.v2Jids.append(node["from"]) - - if node.getChild("enc")["type"] == "pkmsg": + if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG): self.handlePreKeyWhisperMessage(node) - else: + if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG): self.handleWhisperMessage(node) + if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG): + self.handleSenderKeyMessage(node) except InvalidMessageException as e: - # logger.error("Invalid message from %s!! Your axololtl database data might be inconsistent with WhatsApp, or with what that contact has" % node["from"]) - # sys.exit(1) logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMesageNode(node) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) retry.setRegData(self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) except InvalidKeyIdException as e: logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMesageNode(node) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) retry.setRegData(self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) except NoSessionException as e: logger.error(e) - entity = GetKeysIqProtocolEntity([node["from"]]) - if node["from"] not in self.pendingIncomingMessages: - self.pendingIncomingMessages[node["from"]] = [] - self.pendingIncomingMessages[node["from"]].append(node) + entity = GetKeysIqProtocolEntity([senderJid]) + if senderJid not in self.pendingIncomingMessages: + self.pendingIncomingMessages[senderJid] = [] + self.pendingIncomingMessages[senderJid].append(node) self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingIncomingMessages), self.onGetKeysError) except DuplicateMessageException as e: @@ -260,40 +271,88 @@ def handleEncMessage(self, node): def handlePreKeyWhisperMessage(self, node): pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - - preKeyWhisperMessage = PreKeyWhisperMessage(serialized=pkMessageProtocolEntity.getEncData()) - sessionCipher = self.getSessionCipher(pkMessageProtocolEntity.getFrom(False)) + enc = pkMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG) + preKeyWhisperMessage = PreKeyWhisperMessage(serialized=enc.getData()) + sessionCipher = self.getSessionCipher(pkMessageProtocolEntity.getAuthor(False)) plaintext = sessionCipher.decryptPkmsg(preKeyWhisperMessage) - - if pkMessageProtocolEntity.getVersion() == 2: - plaintext = self.unpadV2Plaintext(plaintext) - - - bodyNode = ProtocolTreeNode("body", data = plaintext) - node.addChild(bodyNode) - self.toUpper(node) + if enc.getVersion() == 2: + padding = ord(plaintext[-1]) & 0xFF + self.parseAndHandleMessageProto(pkMessageProtocolEntity, plaintext[:-padding]) + else: + self.handleConversationMessage(node, plaintext) def handleWhisperMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - whisperMessage = WhisperMessage(serialized=encMessageProtocolEntity.getEncData()) - sessionCipher = self.getSessionCipher(encMessageProtocolEntity.getFrom(False)) + enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG) + whisperMessage = WhisperMessage(serialized=enc.getData()) + sessionCipher = self.getSessionCipher(encMessageProtocolEntity.getAuthor(False)) plaintext = sessionCipher.decryptMsg(whisperMessage) - if encMessageProtocolEntity.getVersion() == 2: - plaintext = self.unpadV2Plaintext(plaintext) + if enc.getVersion() == 2: + padding = ord(plaintext[-1]) & 0xFF + self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) + else: + self.handleConversationMessage(encMessageProtocolEntity, plaintext) - bodyNode = ProtocolTreeNode("body", data = plaintext) - node.addChild(bodyNode) - self.toUpper(node) + def handleSenderKeyMessage(self, node): + encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) + enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG) - def unpadV2Plaintext(self, v2plaintext): - if len(v2plaintext) < 128: - return v2plaintext[2:-1] - else: # < 128 * 128 - return v2plaintext[3: -1] + senderKeyName = SenderKeyName(encMessageProtocolEntity.getFrom(True), AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 1)) + groupCipher = GroupCipher(self.store, senderKeyName) + try: + plaintext = groupCipher.decrypt(enc.getData()) + if plaintext: + self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext) + except NoSessionException as e: + logger.error(e) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) + retry.setRegData(self.store.getLocalRegistrationId()) + self.toLower(retry.toProtocolTreeNode()) - #### + def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): + node = encMessageProtocolEntity.toProtocolTreeNode() + m = Message() + try: + m.ParseFromString(serializedData) + except: + print("DUMP:") + print(serializedData) + print([s for s in serializedData]) + print([ord(s) for s in serializedData]) + raise + if not m or not serializedData: + raise ValueError("Empty message") + + handled = False + + if m.senderKeyDistributionMessage is not None: + axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 1) + self.handleSenderKeyDistributionMessage(m.senderKeyDistributionMessage, axolotlAddress) + handled = True + if m.conversation is not None: + self.handleConversationMessage(node, m.conversation) + handled = True + + if not handled: + print(m) + raise ValueError("Unhandled") + + ##TODO other message types + + def handleSenderKeyDistributionMessage(self, senderKeyDistributionMessage, axolotlAddress): + groupId = senderKeyDistributionMessage.groupId + axolotlSenderKeyDistributionMessage = SenderKeyDistributionMessage(serialized=senderKeyDistributionMessage.axolotlSenderKeyDistributionMessage) + groupSessionBuilder = GroupSessionBuilder(self.store) + senderKeyName = SenderKeyName(groupId, axolotlAddress) + groupSessionBuilder.process(senderKeyName, axolotlSenderKeyDistributionMessage) + + def handleConversationMessage(self, originalEncNode, text): + messageNode = copy.deepcopy(originalEncNode) + messageNode.children = [] + messageNode.addChild(ProtocolTreeNode("body", data = text)) + self.toUpper(messageNode) ### keys set and get def sendKeys(self, fresh = True, countPreKeys = _COUNT_PREKEYS): @@ -380,7 +439,18 @@ def adjustId(self, _id): def getSessionCipher(self, recipientId): if recipientId in self.sessionCiphers: - return self.sessionCiphers[recipientId] + sessionCipher = self.sessionCiphers[recipientId] + else: + sessionCipher = SessionCipher(self.store, self.store, self.store, self.store, recipientId, 1) + self.sessionCiphers[recipientId] = sessionCipher + + return sessionCipher + + def getGroupCipher(self, groupId, senderId): + senderKeyName = SenderKeyName(groupId, AxolotlAddress(senderId, 1)) + if senderKeyName in self.groupCiphers: + groupCipher = self.groupCiphers[senderKeyName] else: - self.sessionCiphers[recipientId] = SessionCipher(self.store, self.store, self.store, self.store, recipientId, 1) - return self.sessionCiphers[recipientId] + groupCipher = GroupCipher(self.store, senderKeyName) + self.groupCiphers[senderKeyName] = groupCipher + return groupCipher diff --git a/yowsup/layers/axolotl/protocolentities/__init__.py b/yowsup/layers/axolotl/protocolentities/__init__.py index 59944b87b..8aee2456a 100644 --- a/yowsup/layers/axolotl/protocolentities/__init__.py +++ b/yowsup/layers/axolotl/protocolentities/__init__.py @@ -2,4 +2,6 @@ from .iq_keys_set import SetKeysIqProtocolEntity from .iq_keys_get_result import ResultGetKeysIqProtocolEntity from .message_encrypted import EncryptedMessageProtocolEntity -from .notification_encrypt import EncryptNotification \ No newline at end of file +from .enc import EncProtocolEntity +from .notification_encrypt import EncryptNotification +from .receipt_outgoing_retry import RetryOutgoingReceiptProtocolEntity diff --git a/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py b/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py index 79d51996a..d243efaa3 100644 --- a/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py +++ b/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py @@ -4,7 +4,7 @@ class RetryOutgoingReceiptProtocolEntity(OutgoingReceiptProtocolEntity): ''' - + @@ -14,8 +14,8 @@ class RetryOutgoingReceiptProtocolEntity(OutgoingReceiptProtocolEntity): ''' - def __init__(self, _id, to, t, v = "1", count = "1",regData = ""): - super(RetryOutgoingReceiptProtocolEntity, self).__init__(_id,to) + def __init__(self, _id, to, t, v = "1", count = "1",regData = "", participant = None): + super(RetryOutgoingReceiptProtocolEntity, self).__init__(_id,to, participant=participant) self.setRetryData(t,v,count,regData) def setRetryData(self, t,v,count,regData): @@ -53,10 +53,11 @@ def fromProtocolTreeNode(node): @staticmethod - def fromMesageNode(MessageNodeToBeRetried): + def fromMessageNode(MessageNodeToBeRetried): return RetryOutgoingReceiptProtocolEntity( MessageNodeToBeRetried.getAttributeValue("id"), MessageNodeToBeRetried.getAttributeValue("from"), MessageNodeToBeRetried.getAttributeValue("t"), - MessageNodeToBeRetried.getChild("enc").getAttributeValue("v") - ) \ No newline at end of file + 1, + participant=MessageNodeToBeRetried.getAttributeValue("participant") + ) From bd285a75520c92fee7cb63f74bdcbccc39c66866 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Mon, 18 Jan 2016 19:28:31 +0200 Subject: [PATCH 30/68] Update proto --- yowsup/layers/axolotl/e2e_pb2.py | 218 +++++++++++++------------- yowsup/layers/axolotl/layer.py | 15 +- yowsup/layers/axolotl/proto/e2e.proto | 78 ++++----- 3 files changed, 158 insertions(+), 153 deletions(-) diff --git a/yowsup/layers/axolotl/e2e_pb2.py b/yowsup/layers/axolotl/e2e_pb2.py index 96a45e941..34b2b08ba 100644 --- a/yowsup/layers/axolotl/e2e_pb2.py +++ b/yowsup/layers/axolotl/e2e_pb2.py @@ -17,8 +17,8 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='e2e.proto', - package='', - serialized_pb=_b('\n\te2e.proto\"\xa9\x02\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12\x43\n\x1csenderKeyDistributionMessage\x18\x02 \x01(\x0b\x32\x1d.SenderKeyDistributionMessage\x12#\n\x0cimageMessage\x18\x03 \x01(\x0b\x32\r.ImageMessage\x12\'\n\x0e\x63ontactMessage\x18\x04 \x01(\x0b\x32\x0f.ContactMessage\x12)\n\x0flocationMessage\x18\x05 \x01(\x0b\x32\x10.LocationMessage\x12)\n\x0f\x64ocumentMessage\x18\x07 \x01(\x0b\x32\x10.DocumentMessage\x12\x1f\n\nurlMessage\x18\x06 \x01(\x0b\x32\x0b.UrlMessage\"\\\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x01(\x0c\x12+\n#axolotlSenderKeyDistributionMessage\x18\x02 \x01(\x0c\"\xae\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x01(\x0c\x12\x10\n\x08mimeType\x18\x02 \x01(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x01(\t\x12\x12\n\nfileSha256\x18\x04 \x01(\x0c\x12\x12\n\nfileLength\x18\x05 \x01(\x04\x12\x0e\n\x06height\x18\x06 \x01(\r\x12\r\n\x05width\x18\x07 \x01(\r\x12\x10\n\x08mediaKey\x18\x08 \x01(\x0c\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\x0c\"\x87\x01\n\x0fLocationMessage\x12\x17\n\x0f\x64\x65greesLatitude\x18\x01 \x01(\x01\x12\x18\n\x10\x64\x65greesLongitude\x18\x02 \x01(\x01\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x01(\t\x12\x0b\n\x03url\x18\x05 \x01(\t\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\x0c\"\xa3\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x10\n\x08mimeType\x18\x02 \x01(\t\x12\r\n\x05title\x18\x03 \x01(\t\x12\x12\n\nfileSha256\x18\x04 \x01(\x0c\x12\x12\n\nfileLength\x18\x05 \x01(\x04\x12\x11\n\tpageCount\x18\x06 \x01(\r\x12\x10\n\x08mediaKey\x18\x07 \x01(\x0c\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\x0c\"\x80\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x01(\t\x12\x13\n\x0bmatchedText\x18\x02 \x01(\t\x12\x14\n\x0c\x63\x61nonicalUrl\x18\x04 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05title\x18\x06 \x01(\t\x12\x15\n\rjpegThumbnail\x18\x10 \x01(\t\"4\n\x0e\x43ontactMessage\x12\x13\n\x0b\x64isplayName\x18\x01 \x01(\t\x12\r\n\x05vcard\x18\x10 \x01(\t') + package='com.whatsapp.proto', + serialized_pb=_b('\n\te2e.proto\x12\x12\x63om.whatsapp.proto\"\xa2\x03\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12Y\n\x1fsender_key_distribution_message\x18\x02 \x01(\x0b\x32\x30.com.whatsapp.proto.SenderKeyDistributionMessage\x12\x37\n\rimage_message\x18\x03 \x01(\x0b\x32 .com.whatsapp.proto.ImageMessage\x12;\n\x0f\x63ontact_message\x18\x04 \x01(\x0b\x32\".com.whatsapp.proto.ContactMessage\x12=\n\x10location_message\x18\x05 \x01(\x0b\x32#.com.whatsapp.proto.LocationMessage\x12=\n\x10\x64ocument_message\x18\x07 \x01(\x0b\x32#.com.whatsapp.proto.DocumentMessage\x12\x32\n\nurlMessage\x18\x06 \x01(\x0b\x32\x1e.com.whatsapp.proto.UrlMessage\"`\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x02(\t\x12/\n\'axolotl_sender_key_distribution_message\x18\x02 \x02(\x0c\"\xb3\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x02(\x0c\x12\x11\n\tmime_type\x18\x02 \x02(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x0e\n\x06height\x18\x06 \x02(\r\x12\r\n\x05width\x18\x07 \x02(\r\x12\x11\n\tmedia_key\x18\x08 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x8a\x01\n\x0fLocationMessage\x12\x18\n\x10\x64\x65grees_latitude\x18\x01 \x02(\x01\x12\x19\n\x11\x64\x65grees_longitude\x18\x02 \x02(\x01\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x02(\t\x12\x0b\n\x03url\x18\x05 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\xa8\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x02(\t\x12\x10\n\x08mimeType\x18\x02 \x02(\t\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x12\n\npage_count\x18\x06 \x02(\r\x12\x11\n\tmedia_key\x18\x07 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x83\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x14\n\x0cmatched_text\x18\x02 \x02(\t\x12\x15\n\rcanonical_url\x18\x04 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x02(\t\x12\r\n\x05title\x18\x06 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\t\"5\n\x0e\x43ontactMessage\x12\x14\n\x0c\x64isplay_name\x18\x01 \x02(\t\x12\r\n\x05vcard\x18\x10 \x02(\t') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -27,55 +27,55 @@ _MESSAGE = _descriptor.Descriptor( name='Message', - full_name='Message', + full_name='com.whatsapp.proto.Message', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='conversation', full_name='Message.conversation', index=0, + name='conversation', full_name='com.whatsapp.proto.Message.conversation', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='senderKeyDistributionMessage', full_name='Message.senderKeyDistributionMessage', index=1, + name='sender_key_distribution_message', full_name='com.whatsapp.proto.Message.sender_key_distribution_message', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='imageMessage', full_name='Message.imageMessage', index=2, + name='image_message', full_name='com.whatsapp.proto.Message.image_message', index=2, number=3, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='contactMessage', full_name='Message.contactMessage', index=3, + name='contact_message', full_name='com.whatsapp.proto.Message.contact_message', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='locationMessage', full_name='Message.locationMessage', index=4, + name='location_message', full_name='com.whatsapp.proto.Message.location_message', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='documentMessage', full_name='Message.documentMessage', index=5, + name='document_message', full_name='com.whatsapp.proto.Message.document_message', index=5, number=7, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='urlMessage', full_name='Message.urlMessage', index=6, + name='urlMessage', full_name='com.whatsapp.proto.Message.urlMessage', index=6, number=6, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -92,28 +92,28 @@ extension_ranges=[], oneofs=[ ], - serialized_start=14, - serialized_end=311, + serialized_start=34, + serialized_end=452, ) _SENDERKEYDISTRIBUTIONMESSAGE = _descriptor.Descriptor( name='SenderKeyDistributionMessage', - full_name='SenderKeyDistributionMessage', + full_name='com.whatsapp.proto.SenderKeyDistributionMessage', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='groupId', full_name='SenderKeyDistributionMessage.groupId', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), + name='groupId', full_name='com.whatsapp.proto.SenderKeyDistributionMessage.groupId', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='axolotlSenderKeyDistributionMessage', full_name='SenderKeyDistributionMessage.axolotlSenderKeyDistributionMessage', index=1, - number=2, type=12, cpp_type=9, label=1, + name='axolotl_sender_key_distribution_message', full_name='com.whatsapp.proto.SenderKeyDistributionMessage.axolotl_sender_key_distribution_message', index=1, + number=2, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -129,77 +129,77 @@ extension_ranges=[], oneofs=[ ], - serialized_start=313, - serialized_end=405, + serialized_start=454, + serialized_end=550, ) _IMAGEMESSAGE = _descriptor.Descriptor( name='ImageMessage', - full_name='ImageMessage', + full_name='com.whatsapp.proto.ImageMessage', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='url', full_name='ImageMessage.url', index=0, - number=1, type=12, cpp_type=9, label=1, + name='url', full_name='com.whatsapp.proto.ImageMessage.url', index=0, + number=1, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='mimeType', full_name='ImageMessage.mimeType', index=1, - number=2, type=9, cpp_type=9, label=1, + name='mime_type', full_name='com.whatsapp.proto.ImageMessage.mime_type', index=1, + number=2, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='caption', full_name='ImageMessage.caption', index=2, - number=3, type=9, cpp_type=9, label=1, + name='caption', full_name='com.whatsapp.proto.ImageMessage.caption', index=2, + number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='fileSha256', full_name='ImageMessage.fileSha256', index=3, - number=4, type=12, cpp_type=9, label=1, + name='file_sha256', full_name='com.whatsapp.proto.ImageMessage.file_sha256', index=3, + number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='fileLength', full_name='ImageMessage.fileLength', index=4, - number=5, type=4, cpp_type=4, label=1, + name='file_length', full_name='com.whatsapp.proto.ImageMessage.file_length', index=4, + number=5, type=4, cpp_type=4, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='height', full_name='ImageMessage.height', index=5, - number=6, type=13, cpp_type=3, label=1, + name='height', full_name='com.whatsapp.proto.ImageMessage.height', index=5, + number=6, type=13, cpp_type=3, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='width', full_name='ImageMessage.width', index=6, - number=7, type=13, cpp_type=3, label=1, + name='width', full_name='com.whatsapp.proto.ImageMessage.width', index=6, + number=7, type=13, cpp_type=3, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='mediaKey', full_name='ImageMessage.mediaKey', index=7, - number=8, type=12, cpp_type=9, label=1, + name='media_key', full_name='com.whatsapp.proto.ImageMessage.media_key', index=7, + number=8, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='jpegThumbnail', full_name='ImageMessage.jpegThumbnail', index=8, - number=16, type=12, cpp_type=9, label=1, + name='jpeg_thumbnail', full_name='com.whatsapp.proto.ImageMessage.jpeg_thumbnail', index=8, + number=16, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -215,56 +215,56 @@ extension_ranges=[], oneofs=[ ], - serialized_start=408, - serialized_end=582, + serialized_start=553, + serialized_end=732, ) _LOCATIONMESSAGE = _descriptor.Descriptor( name='LocationMessage', - full_name='LocationMessage', + full_name='com.whatsapp.proto.LocationMessage', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='degreesLatitude', full_name='LocationMessage.degreesLatitude', index=0, - number=1, type=1, cpp_type=5, label=1, + name='degrees_latitude', full_name='com.whatsapp.proto.LocationMessage.degrees_latitude', index=0, + number=1, type=1, cpp_type=5, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='degreesLongitude', full_name='LocationMessage.degreesLongitude', index=1, - number=2, type=1, cpp_type=5, label=1, + name='degrees_longitude', full_name='com.whatsapp.proto.LocationMessage.degrees_longitude', index=1, + number=2, type=1, cpp_type=5, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='name', full_name='LocationMessage.name', index=2, - number=3, type=9, cpp_type=9, label=1, + name='name', full_name='com.whatsapp.proto.LocationMessage.name', index=2, + number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='address', full_name='LocationMessage.address', index=3, - number=4, type=9, cpp_type=9, label=1, + name='address', full_name='com.whatsapp.proto.LocationMessage.address', index=3, + number=4, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='url', full_name='LocationMessage.url', index=4, - number=5, type=9, cpp_type=9, label=1, + name='url', full_name='com.whatsapp.proto.LocationMessage.url', index=4, + number=5, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='jpegThumbnail', full_name='LocationMessage.jpegThumbnail', index=5, - number=16, type=12, cpp_type=9, label=1, + name='jpeg_thumbnail', full_name='com.whatsapp.proto.LocationMessage.jpeg_thumbnail', index=5, + number=16, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -280,70 +280,70 @@ extension_ranges=[], oneofs=[ ], - serialized_start=585, - serialized_end=720, + serialized_start=735, + serialized_end=873, ) _DOCUMENTMESSAGE = _descriptor.Descriptor( name='DocumentMessage', - full_name='DocumentMessage', + full_name='com.whatsapp.proto.DocumentMessage', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='url', full_name='DocumentMessage.url', index=0, - number=1, type=9, cpp_type=9, label=1, + name='url', full_name='com.whatsapp.proto.DocumentMessage.url', index=0, + number=1, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='mimeType', full_name='DocumentMessage.mimeType', index=1, - number=2, type=9, cpp_type=9, label=1, + name='mimeType', full_name='com.whatsapp.proto.DocumentMessage.mimeType', index=1, + number=2, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='title', full_name='DocumentMessage.title', index=2, - number=3, type=9, cpp_type=9, label=1, + name='title', full_name='com.whatsapp.proto.DocumentMessage.title', index=2, + number=3, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='fileSha256', full_name='DocumentMessage.fileSha256', index=3, - number=4, type=12, cpp_type=9, label=1, + name='file_sha256', full_name='com.whatsapp.proto.DocumentMessage.file_sha256', index=3, + number=4, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='fileLength', full_name='DocumentMessage.fileLength', index=4, - number=5, type=4, cpp_type=4, label=1, + name='file_length', full_name='com.whatsapp.proto.DocumentMessage.file_length', index=4, + number=5, type=4, cpp_type=4, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='pageCount', full_name='DocumentMessage.pageCount', index=5, - number=6, type=13, cpp_type=3, label=1, + name='page_count', full_name='com.whatsapp.proto.DocumentMessage.page_count', index=5, + number=6, type=13, cpp_type=3, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='mediaKey', full_name='DocumentMessage.mediaKey', index=6, - number=7, type=12, cpp_type=9, label=1, + name='media_key', full_name='com.whatsapp.proto.DocumentMessage.media_key', index=6, + number=7, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='jpegThumbnail', full_name='DocumentMessage.jpegThumbnail', index=7, - number=16, type=12, cpp_type=9, label=1, + name='jpeg_thumbnail', full_name='com.whatsapp.proto.DocumentMessage.jpeg_thumbnail', index=7, + number=16, type=12, cpp_type=9, label=2, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -359,56 +359,56 @@ extension_ranges=[], oneofs=[ ], - serialized_start=723, - serialized_end=886, + serialized_start=876, + serialized_end=1044, ) _URLMESSAGE = _descriptor.Descriptor( name='UrlMessage', - full_name='UrlMessage', + full_name='com.whatsapp.proto.UrlMessage', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='text', full_name='UrlMessage.text', index=0, - number=1, type=9, cpp_type=9, label=1, + name='text', full_name='com.whatsapp.proto.UrlMessage.text', index=0, + number=1, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='matchedText', full_name='UrlMessage.matchedText', index=1, - number=2, type=9, cpp_type=9, label=1, + name='matched_text', full_name='com.whatsapp.proto.UrlMessage.matched_text', index=1, + number=2, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='canonicalUrl', full_name='UrlMessage.canonicalUrl', index=2, - number=4, type=9, cpp_type=9, label=1, + name='canonical_url', full_name='com.whatsapp.proto.UrlMessage.canonical_url', index=2, + number=4, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='description', full_name='UrlMessage.description', index=3, - number=5, type=9, cpp_type=9, label=1, + name='description', full_name='com.whatsapp.proto.UrlMessage.description', index=3, + number=5, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='title', full_name='UrlMessage.title', index=4, - number=6, type=9, cpp_type=9, label=1, + name='title', full_name='com.whatsapp.proto.UrlMessage.title', index=4, + number=6, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='jpegThumbnail', full_name='UrlMessage.jpegThumbnail', index=5, - number=16, type=9, cpp_type=9, label=1, + name='jpeg_thumbnail', full_name='com.whatsapp.proto.UrlMessage.jpeg_thumbnail', index=5, + number=16, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -424,28 +424,28 @@ extension_ranges=[], oneofs=[ ], - serialized_start=889, - serialized_end=1017, + serialized_start=1047, + serialized_end=1178, ) _CONTACTMESSAGE = _descriptor.Descriptor( name='ContactMessage', - full_name='ContactMessage', + full_name='com.whatsapp.proto.ContactMessage', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='displayName', full_name='ContactMessage.displayName', index=0, - number=1, type=9, cpp_type=9, label=1, + name='display_name', full_name='com.whatsapp.proto.ContactMessage.display_name', index=0, + number=1, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='vcard', full_name='ContactMessage.vcard', index=1, - number=16, type=9, cpp_type=9, label=1, + name='vcard', full_name='com.whatsapp.proto.ContactMessage.vcard', index=1, + number=16, type=9, cpp_type=9, label=2, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, @@ -461,15 +461,15 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1019, - serialized_end=1071, + serialized_start=1180, + serialized_end=1233, ) -_MESSAGE.fields_by_name['senderKeyDistributionMessage'].message_type = _SENDERKEYDISTRIBUTIONMESSAGE -_MESSAGE.fields_by_name['imageMessage'].message_type = _IMAGEMESSAGE -_MESSAGE.fields_by_name['contactMessage'].message_type = _CONTACTMESSAGE -_MESSAGE.fields_by_name['locationMessage'].message_type = _LOCATIONMESSAGE -_MESSAGE.fields_by_name['documentMessage'].message_type = _DOCUMENTMESSAGE +_MESSAGE.fields_by_name['sender_key_distribution_message'].message_type = _SENDERKEYDISTRIBUTIONMESSAGE +_MESSAGE.fields_by_name['image_message'].message_type = _IMAGEMESSAGE +_MESSAGE.fields_by_name['contact_message'].message_type = _CONTACTMESSAGE +_MESSAGE.fields_by_name['location_message'].message_type = _LOCATIONMESSAGE +_MESSAGE.fields_by_name['document_message'].message_type = _DOCUMENTMESSAGE _MESSAGE.fields_by_name['urlMessage'].message_type = _URLMESSAGE DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE DESCRIPTOR.message_types_by_name['SenderKeyDistributionMessage'] = _SENDERKEYDISTRIBUTIONMESSAGE @@ -482,49 +482,49 @@ Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( DESCRIPTOR = _MESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:Message) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.Message) )) _sym_db.RegisterMessage(Message) SenderKeyDistributionMessage = _reflection.GeneratedProtocolMessageType('SenderKeyDistributionMessage', (_message.Message,), dict( DESCRIPTOR = _SENDERKEYDISTRIBUTIONMESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:SenderKeyDistributionMessage) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.SenderKeyDistributionMessage) )) _sym_db.RegisterMessage(SenderKeyDistributionMessage) ImageMessage = _reflection.GeneratedProtocolMessageType('ImageMessage', (_message.Message,), dict( DESCRIPTOR = _IMAGEMESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:ImageMessage) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.ImageMessage) )) _sym_db.RegisterMessage(ImageMessage) LocationMessage = _reflection.GeneratedProtocolMessageType('LocationMessage', (_message.Message,), dict( DESCRIPTOR = _LOCATIONMESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:LocationMessage) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.LocationMessage) )) _sym_db.RegisterMessage(LocationMessage) DocumentMessage = _reflection.GeneratedProtocolMessageType('DocumentMessage', (_message.Message,), dict( DESCRIPTOR = _DOCUMENTMESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:DocumentMessage) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.DocumentMessage) )) _sym_db.RegisterMessage(DocumentMessage) UrlMessage = _reflection.GeneratedProtocolMessageType('UrlMessage', (_message.Message,), dict( DESCRIPTOR = _URLMESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:UrlMessage) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.UrlMessage) )) _sym_db.RegisterMessage(UrlMessage) ContactMessage = _reflection.GeneratedProtocolMessageType('ContactMessage', (_message.Message,), dict( DESCRIPTOR = _CONTACTMESSAGE, __module__ = 'e2e_pb2' - # @@protoc_insertion_point(class_scope:ContactMessage) + # @@protoc_insertion_point(class_scope:com.whatsapp.proto.ContactMessage) )) _sym_db.RegisterMessage(ContactMessage) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index c21fcd037..6bf76c29e 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -299,12 +299,14 @@ def handleSenderKeyMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG) - senderKeyName = SenderKeyName(encMessageProtocolEntity.getFrom(True), AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 1)) + senderKeyName = SenderKeyName(encMessageProtocolEntity.getFrom(True), AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0)) groupCipher = GroupCipher(self.store, senderKeyName) try: plaintext = groupCipher.decrypt(enc.getData()) if plaintext: self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext) + else: + self.handleConversationMessage(node, "NONE22!!") except NoSessionException as e: logger.error(e) retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) @@ -321,15 +323,16 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): print(serializedData) print([s for s in serializedData]) print([ord(s) for s in serializedData]) - raise + self.handleConversationMessage(node, "NONE!!") + return if not m or not serializedData: raise ValueError("Empty message") handled = False - if m.senderKeyDistributionMessage is not None: - axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 1) - self.handleSenderKeyDistributionMessage(m.senderKeyDistributionMessage, axolotlAddress) + if encMessageProtocolEntity.isGroupMessage() and m.sender_key_distribution_message is not None: + axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) + self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) handled = True if m.conversation is not None: self.handleConversationMessage(node, m.conversation) @@ -343,7 +346,7 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): def handleSenderKeyDistributionMessage(self, senderKeyDistributionMessage, axolotlAddress): groupId = senderKeyDistributionMessage.groupId - axolotlSenderKeyDistributionMessage = SenderKeyDistributionMessage(serialized=senderKeyDistributionMessage.axolotlSenderKeyDistributionMessage) + axolotlSenderKeyDistributionMessage = SenderKeyDistributionMessage(serialized=senderKeyDistributionMessage.axolotl_sender_key_distribution_message) groupSessionBuilder = GroupSessionBuilder(self.store) senderKeyName = SenderKeyName(groupId, axolotlAddress) groupSessionBuilder.process(senderKeyName, axolotlSenderKeyDistributionMessage) diff --git a/yowsup/layers/axolotl/proto/e2e.proto b/yowsup/layers/axolotl/proto/e2e.proto index c99508197..22c8ebec3 100644 --- a/yowsup/layers/axolotl/proto/e2e.proto +++ b/yowsup/layers/axolotl/proto/e2e.proto @@ -1,62 +1,64 @@ +package com.whatsapp.proto; + message Message { optional string conversation = 1; - optional SenderKeyDistributionMessage senderKeyDistributionMessage = 2; - optional ImageMessage imageMessage = 3; - optional ContactMessage contactMessage = 4; - optional LocationMessage locationMessage = 5; - optional DocumentMessage documentMessage = 7; + optional SenderKeyDistributionMessage sender_key_distribution_message = 2; + optional ImageMessage image_message = 3; + optional ContactMessage contact_message = 4; + optional LocationMessage location_message = 5; + optional DocumentMessage document_message = 7; optional UrlMessage urlMessage = 6; } message SenderKeyDistributionMessage { - optional bytes groupId = 1; - optional bytes axolotlSenderKeyDistributionMessage = 2; + required string groupId = 1; + required bytes axolotl_sender_key_distribution_message = 2; } message ImageMessage { - optional bytes url = 1; - optional string mimeType = 2; - optional string caption = 3; - optional bytes fileSha256 = 4; - optional uint64 fileLength = 5; - optional uint32 height = 6; - optional uint32 width = 7; - optional bytes mediaKey = 8; - optional bytes jpegThumbnail = 16; + required bytes url = 1; + required string mime_type = 2; + required string caption = 3; + required bytes file_sha256 = 4; + required uint64 file_length = 5; + required uint32 height = 6; + required uint32 width = 7; + required bytes media_key = 8; + required bytes jpeg_thumbnail = 16; } message LocationMessage { - optional double degreesLatitude = 1; - optional double degreesLongitude = 2; - optional string name = 3; - optional string address = 4; - optional string url = 5; - optional bytes jpegThumbnail = 16; + required double degrees_latitude = 1; + required double degrees_longitude = 2; + required string name = 3; + required string address = 4; + required string url = 5; + required bytes jpeg_thumbnail = 16; } message DocumentMessage { - optional string url = 1; - optional string mimeType = 2; - optional string title = 3; - optional bytes fileSha256 = 4; - optional uint64 fileLength = 5; - optional uint32 pageCount = 6; - optional bytes mediaKey = 7; - optional bytes jpegThumbnail = 16; + required string url = 1; + required string mimeType = 2; + required string title = 3; + required bytes file_sha256 = 4; + required uint64 file_length = 5; + required uint32 page_count = 6; + required bytes media_key = 7; + required bytes jpeg_thumbnail = 16; } message UrlMessage { - optional string text = 1; - optional string matchedText = 2; - optional string canonicalUrl = 4; - optional string description = 5; - optional string title = 6; - optional string jpegThumbnail = 16; + required string text = 1; + required string matched_text = 2; + required string canonical_url = 4; + required string description = 5; + required string title = 6; + required string jpeg_thumbnail = 16; } message ContactMessage { - optional string displayName = 1; - optional string vcard = 16; + required string display_name = 1; + required string vcard = 16; } From 8fdbb834cef0be6b9e98441d37e5ba9cfac96304 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 19 Jan 2016 04:14:48 +0200 Subject: [PATCH 31/68] Unpad skmsg --- yowsup/layers/axolotl/layer.py | 13 +++++-------- yowsup/layers/axolotl/protocolentities/enc.py | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index 6bf76c29e..a788ca951 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -303,10 +303,8 @@ def handleSenderKeyMessage(self, node): groupCipher = GroupCipher(self.store, senderKeyName) try: plaintext = groupCipher.decrypt(enc.getData()) - if plaintext: - self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext) - else: - self.handleConversationMessage(node, "NONE22!!") + padding = ord(plaintext[-1]) & 0xFF + self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) except NoSessionException as e: logger.error(e) retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) @@ -323,18 +321,17 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): print(serializedData) print([s for s in serializedData]) print([ord(s) for s in serializedData]) - self.handleConversationMessage(node, "NONE!!") - return + raise if not m or not serializedData: raise ValueError("Empty message") handled = False - if encMessageProtocolEntity.isGroupMessage() and m.sender_key_distribution_message is not None: + if m.HasField("sender_key_distribution_message"): axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) handled = True - if m.conversation is not None: + if m.HasField("conversation"): self.handleConversationMessage(node, m.conversation) handled = True diff --git a/yowsup/layers/axolotl/protocolentities/enc.py b/yowsup/layers/axolotl/protocolentities/enc.py index 5aea2608c..b181fa0a0 100644 --- a/yowsup/layers/axolotl/protocolentities/enc.py +++ b/yowsup/layers/axolotl/protocolentities/enc.py @@ -1,5 +1,5 @@ from yowsup.structs import ProtocolEntity, ProtocolTreeNode - +import sys class EncProtocolEntity(ProtocolEntity): TYPE_PKMSG = "pkmsg" TYPE_MSG = "msg" @@ -26,4 +26,4 @@ def toProtocolTreeNode(self): @staticmethod def fromProtocolTreeNode(node): - return EncProtocolEntity(node["type"], node["v"], node.data) + return EncProtocolEntity(node["type"], node["v"], node.data.encode('latin-1') if sys.version_info >= (3,0) else node.data) From ff06c3273f07773e7a333ef7eb7f9a98d999ca66 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 19 Jan 2016 04:44:52 +0200 Subject: [PATCH 32/68] Fixed url_message field name --- yowsup/layers/axolotl/e2e_pb2.py | 32 +++++++++++++-------------- yowsup/layers/axolotl/proto/e2e.proto | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/yowsup/layers/axolotl/e2e_pb2.py b/yowsup/layers/axolotl/e2e_pb2.py index 34b2b08ba..dbef983ba 100644 --- a/yowsup/layers/axolotl/e2e_pb2.py +++ b/yowsup/layers/axolotl/e2e_pb2.py @@ -18,7 +18,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='e2e.proto', package='com.whatsapp.proto', - serialized_pb=_b('\n\te2e.proto\x12\x12\x63om.whatsapp.proto\"\xa2\x03\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12Y\n\x1fsender_key_distribution_message\x18\x02 \x01(\x0b\x32\x30.com.whatsapp.proto.SenderKeyDistributionMessage\x12\x37\n\rimage_message\x18\x03 \x01(\x0b\x32 .com.whatsapp.proto.ImageMessage\x12;\n\x0f\x63ontact_message\x18\x04 \x01(\x0b\x32\".com.whatsapp.proto.ContactMessage\x12=\n\x10location_message\x18\x05 \x01(\x0b\x32#.com.whatsapp.proto.LocationMessage\x12=\n\x10\x64ocument_message\x18\x07 \x01(\x0b\x32#.com.whatsapp.proto.DocumentMessage\x12\x32\n\nurlMessage\x18\x06 \x01(\x0b\x32\x1e.com.whatsapp.proto.UrlMessage\"`\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x02(\t\x12/\n\'axolotl_sender_key_distribution_message\x18\x02 \x02(\x0c\"\xb3\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x02(\x0c\x12\x11\n\tmime_type\x18\x02 \x02(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x0e\n\x06height\x18\x06 \x02(\r\x12\r\n\x05width\x18\x07 \x02(\r\x12\x11\n\tmedia_key\x18\x08 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x8a\x01\n\x0fLocationMessage\x12\x18\n\x10\x64\x65grees_latitude\x18\x01 \x02(\x01\x12\x19\n\x11\x64\x65grees_longitude\x18\x02 \x02(\x01\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x02(\t\x12\x0b\n\x03url\x18\x05 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\xa8\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x02(\t\x12\x10\n\x08mimeType\x18\x02 \x02(\t\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x12\n\npage_count\x18\x06 \x02(\r\x12\x11\n\tmedia_key\x18\x07 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x83\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x14\n\x0cmatched_text\x18\x02 \x02(\t\x12\x15\n\rcanonical_url\x18\x04 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x02(\t\x12\r\n\x05title\x18\x06 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\t\"5\n\x0e\x43ontactMessage\x12\x14\n\x0c\x64isplay_name\x18\x01 \x02(\t\x12\r\n\x05vcard\x18\x10 \x02(\t') + serialized_pb=_b('\n\te2e.proto\x12\x12\x63om.whatsapp.proto\"\xa3\x03\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12Y\n\x1fsender_key_distribution_message\x18\x02 \x01(\x0b\x32\x30.com.whatsapp.proto.SenderKeyDistributionMessage\x12\x37\n\rimage_message\x18\x03 \x01(\x0b\x32 .com.whatsapp.proto.ImageMessage\x12;\n\x0f\x63ontact_message\x18\x04 \x01(\x0b\x32\".com.whatsapp.proto.ContactMessage\x12=\n\x10location_message\x18\x05 \x01(\x0b\x32#.com.whatsapp.proto.LocationMessage\x12=\n\x10\x64ocument_message\x18\x07 \x01(\x0b\x32#.com.whatsapp.proto.DocumentMessage\x12\x33\n\x0burl_message\x18\x06 \x01(\x0b\x32\x1e.com.whatsapp.proto.UrlMessage\"`\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x02(\t\x12/\n\'axolotl_sender_key_distribution_message\x18\x02 \x02(\x0c\"\xb3\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x02(\x0c\x12\x11\n\tmime_type\x18\x02 \x02(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x0e\n\x06height\x18\x06 \x02(\r\x12\r\n\x05width\x18\x07 \x02(\r\x12\x11\n\tmedia_key\x18\x08 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x8a\x01\n\x0fLocationMessage\x12\x18\n\x10\x64\x65grees_latitude\x18\x01 \x02(\x01\x12\x19\n\x11\x64\x65grees_longitude\x18\x02 \x02(\x01\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x02(\t\x12\x0b\n\x03url\x18\x05 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\xa8\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x02(\t\x12\x10\n\x08mimeType\x18\x02 \x02(\t\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x12\n\npage_count\x18\x06 \x02(\r\x12\x11\n\tmedia_key\x18\x07 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x83\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x14\n\x0cmatched_text\x18\x02 \x02(\t\x12\x15\n\rcanonical_url\x18\x04 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x02(\t\x12\r\n\x05title\x18\x06 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\t\"5\n\x0e\x43ontactMessage\x12\x14\n\x0c\x64isplay_name\x18\x01 \x02(\t\x12\r\n\x05vcard\x18\x10 \x02(\t') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -75,7 +75,7 @@ is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='urlMessage', full_name='com.whatsapp.proto.Message.urlMessage', index=6, + name='url_message', full_name='com.whatsapp.proto.Message.url_message', index=6, number=6, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -93,7 +93,7 @@ oneofs=[ ], serialized_start=34, - serialized_end=452, + serialized_end=453, ) @@ -129,8 +129,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=454, - serialized_end=550, + serialized_start=455, + serialized_end=551, ) @@ -215,8 +215,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=553, - serialized_end=732, + serialized_start=554, + serialized_end=733, ) @@ -280,8 +280,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=735, - serialized_end=873, + serialized_start=736, + serialized_end=874, ) @@ -359,8 +359,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=876, - serialized_end=1044, + serialized_start=877, + serialized_end=1045, ) @@ -424,8 +424,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1047, - serialized_end=1178, + serialized_start=1048, + serialized_end=1179, ) @@ -461,8 +461,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1180, - serialized_end=1233, + serialized_start=1181, + serialized_end=1234, ) _MESSAGE.fields_by_name['sender_key_distribution_message'].message_type = _SENDERKEYDISTRIBUTIONMESSAGE @@ -470,7 +470,7 @@ _MESSAGE.fields_by_name['contact_message'].message_type = _CONTACTMESSAGE _MESSAGE.fields_by_name['location_message'].message_type = _LOCATIONMESSAGE _MESSAGE.fields_by_name['document_message'].message_type = _DOCUMENTMESSAGE -_MESSAGE.fields_by_name['urlMessage'].message_type = _URLMESSAGE +_MESSAGE.fields_by_name['url_message'].message_type = _URLMESSAGE DESCRIPTOR.message_types_by_name['Message'] = _MESSAGE DESCRIPTOR.message_types_by_name['SenderKeyDistributionMessage'] = _SENDERKEYDISTRIBUTIONMESSAGE DESCRIPTOR.message_types_by_name['ImageMessage'] = _IMAGEMESSAGE diff --git a/yowsup/layers/axolotl/proto/e2e.proto b/yowsup/layers/axolotl/proto/e2e.proto index 22c8ebec3..4ee6632e1 100644 --- a/yowsup/layers/axolotl/proto/e2e.proto +++ b/yowsup/layers/axolotl/proto/e2e.proto @@ -7,7 +7,7 @@ message Message { optional ContactMessage contact_message = 4; optional LocationMessage location_message = 5; optional DocumentMessage document_message = 7; - optional UrlMessage urlMessage = 6; + optional UrlMessage url_message = 6; } message SenderKeyDistributionMessage { From e751ccdc680523d033e2e17c2102498aff26bcaf Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Tue, 19 Jan 2016 04:45:33 +0200 Subject: [PATCH 33/68] Support recv encrypted image message initial commit --- yowsup/layers/axolotl/layer.py | 63 ++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index a788ca951..4cf930289 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -1,6 +1,4 @@ from yowsup.layers import YowProtocolLayer, YowLayerEvent, EventCallback -from yowsup.layers.protocol_messages.protocolentities.message import MessageProtocolEntity -from yowsup.layers.protocol_messages.protocolentities.message_text import TextMessageProtocolEntity from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity from yowsup.layers.network.layer import YowNetworkLayer from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer @@ -177,8 +175,7 @@ def processPendingIncomingMessages(self, jid): def handlePlaintextNode(self, node): plaintext = node.getChild("body").getData() - entity = MessageProtocolEntity.fromProtocolTreeNode(node) - recipient_id = entity.getTo(False) + recipient_id = node["to"].split('@')[0] if not self.store.containsSession(recipient_id, 1): entity = GetKeysIqProtocolEntity([node["to"]]) @@ -207,7 +204,7 @@ def handlePlaintextNode(self, node): EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , version, ciphertext.serialize())], - MessageProtocolEntity.MESSAGE_TYPE_TEXT, + "text", _id= node["id"], to = node["to"], notify = node["notify"], @@ -238,7 +235,7 @@ def handleEncMessage(self, node): try: if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG): self.handlePreKeyWhisperMessage(node) - if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG): + elif encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG): self.handleWhisperMessage(node) if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG): self.handleSenderKeyMessage(node) @@ -325,22 +322,23 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): if not m or not serializedData: raise ValueError("Empty message") - handled = False - if m.HasField("sender_key_distribution_message"): axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) - handled = True if m.HasField("conversation"): self.handleConversationMessage(node, m.conversation) - handled = True - - if not handled: + elif m.HasField("contact_message"): + self.handleContactMessage(node, m.contact_message) + elif m.HasField("url_message"): + self.handleUrlMessage(node, m.url_message) + elif m.HasField("location_message"): + self.handleLocationMessage(node, m.location_message) + elif m.HasField("image_message"): + self.handleImageMessage(node, m.image_message) + else: print(m) raise ValueError("Unhandled") - ##TODO other message types - def handleSenderKeyDistributionMessage(self, senderKeyDistributionMessage, axolotlAddress): groupId = senderKeyDistributionMessage.groupId axolotlSenderKeyDistributionMessage = SenderKeyDistributionMessage(serialized=senderKeyDistributionMessage.axolotl_sender_key_distribution_message) @@ -354,6 +352,43 @@ def handleConversationMessage(self, originalEncNode, text): messageNode.addChild(ProtocolTreeNode("body", data = text)) self.toUpper(messageNode) + + def handleImageMessage(self, originalEncNode, imageMessage): + messageNode = copy.deepcopy(originalEncNode) + messageNode["type"] = "media" + mediaNode = ProtocolTreeNode("media", { + "type": "image", + "filehash": imageMessage.file_sha256, + "size": str(imageMessage.file_length), + "url": imageMessage.url, + "mimetype": imageMessage.mime_type, + "width": imageMessage.width, + "height": imageMessage.height, + "caption": imageMessage.caption, + "encoding": "raw", + "file": "enc", + "ip": "0" + }, data = imageMessage.jpeg_thumbnail) + messageNode.addChild(mediaNode) + + self.toUpper(messageNode) + + def handleUrlMessage(self, originalEncNode, urlMessage): + #convert to ?? + pass + + def handleLocationMessage(self, originalEncNode, locationMessage): + #convert to LocationMediaMessage + pass + + def handleDocumentMessage(self, originalEncNode, documentMessage): + #convert to ?? + pass + + def handleContactMessage(self, originalEncNode, contactMessage): + #convert to vcardmediamessage + pass + ### keys set and get def sendKeys(self, fresh = True, countPreKeys = _COUNT_PREKEYS): identityKeyPair = KeyHelper.generateIdentityKeyPair() if fresh else self.store.getIdentityKeyPair() From 957db8d20a779e01e865c1d24f29b81b8a6840a6 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Wed, 20 Jan 2016 17:17:31 +0200 Subject: [PATCH 34/68] Protobuf v2 messages initial commit --- yowsup/layers/axolotl/layer.py | 142 +++++++++++++----- yowsup/layers/axolotl/protocolentities/enc.py | 13 +- 2 files changed, 114 insertions(+), 41 deletions(-) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index 4cf930289..b4e4ac717 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -3,7 +3,7 @@ from yowsup.layers.network.layer import YowNetworkLayer from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity -from yowsup.layers.axolotl.e2e_pb2 import Message +from yowsup.layers.axolotl.e2e_pb2 import * from yowsup.layers.axolotl.store.sqlite.liteaxolotlstore import LiteAxolotlStore from yowsup.layers.axolotl.protocolentities import * from yowsup.structs import ProtocolTreeNode @@ -115,7 +115,7 @@ def onDisconnected(self, yowLayerEvent): self.store = None def send(self, node): - if node.tag == "message" and node["type"] == "text" and node["to"] not in self.skipEncJids: + if node.tag == "message" and node["to"] not in self.skipEncJids: self.handlePlaintextNode(node) return self.toLower(node) @@ -174,7 +174,6 @@ def processPendingIncomingMessages(self, jid): #### handling message types def handlePlaintextNode(self, node): - plaintext = node.getChild("body").getData() recipient_id = node["to"].split('@')[0] if not self.store.containsSession(recipient_id, 1): @@ -185,26 +184,20 @@ def handlePlaintextNode(self, node): self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingMessages), self.onGetKeysError) else: + message = None + messagebytes = self.serializeToProtobuf(node) + if messagebytes: + sessionCipher = self.getSessionCipher(recipient_id) + ciphertext = sessionCipher.encrypt(messagebytes) - sessionCipher = self.getSessionCipher(recipient_id) - - if node["to"] in self.v2Jids: - version = 2 - padded = bytearray() - padded.append(ord("\n")) - padded.extend(self.encodeInt7bit(len(plaintext))) - padded.extend(plaintext) - padded.append(ord("\x01")) - plaintext = padded - else: - version = 1 - ciphertext = sessionCipher.encrypt(plaintext) - encEntity = EncryptedMessageProtocolEntity( + mediaType = node.getChild("media")["type"] if node.getChild("media") else None + + encEntity = EncryptedMessageProtocolEntity( [ EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , - version, - ciphertext.serialize())], - "text", + 2, + ciphertext.serialize(), mediaType)], + "text" if not mediaType else "media", _id= node["id"], to = node["to"], notify = node["notify"], @@ -213,17 +206,9 @@ def handlePlaintextNode(self, node): offline=node["offline"], retry=node["retry"] ) - self.toLower(encEntity.toProtocolTreeNode()) - - def encodeInt7bit(self, value): - v = value - out = bytearray() - while v >= 0x80: - out.append((v | 0x80) % 256) - v >>= 7 - out.append(v % 256) - - return out + self.toLower(encEntity.toProtocolTreeNode()) + else: #case of unserializable messages (audio, video) ? + self.toLower(node) def handleEncMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) @@ -352,7 +337,6 @@ def handleConversationMessage(self, originalEncNode, text): messageNode.addChild(ProtocolTreeNode("body", data = text)) self.toUpper(messageNode) - def handleImageMessage(self, originalEncNode, imageMessage): messageNode = copy.deepcopy(originalEncNode) messageNode["type"] = "media" @@ -377,16 +361,99 @@ def handleUrlMessage(self, originalEncNode, urlMessage): #convert to ?? pass - def handleLocationMessage(self, originalEncNode, locationMessage): - #convert to LocationMediaMessage - pass - def handleDocumentMessage(self, originalEncNode, documentMessage): #convert to ?? pass + def handleLocationMessage(self, originalEncNode, locationMessage): + messageNode = copy.deepycopy(originalEncNode) + messageNode["type"] = "media" + mediaNode = ProtocolTreeNode("media", { + "latitude": locationMessage.degrees_latitude, + "longitude": locationMessage.degress_longitude, + "name": "%s %s" % (locationMessage.name, locationMessage.address), + "url": locationMessage.url, + "encoding": "raw", + "type": "location" + }, data=locationMessage.jpeg_thumbnail) + messageNode.addChild(mediaNode) + self.toUpper(messageNode) + def handleContactMessage(self, originalEncNode, contactMessage): - #convert to vcardmediamessage + messageNode = copy.deepycopy(originalEncNode) + messageNode["type"] = "media" + mediaNode = ProtocolTreeNode("media", { + "type": "vcard" + }, [ + ProtocolTreeNode("vcard", {"name": contactMessage.display_name}, data = contactMessage.vcard) + ] ) + messageNode.addChild(mediaNode) + self.toUpper(messageNode) + + def serializeToProtobuf(self, node): + if node.getChild("body"): + return self.serializeTextToProtobuf(node) + elif node.getChild("media"): + return self.serializeMediaToProtobuf(node.getChild("media")) + else: + raise ValueError("No body or media nodes found") + + def serializeTextToProtobuf(self, node): + m = Message() + m.conversation = node.getChild("body").getData() + return m.SerializeToString() + + def serializeMediaToProtobuf(self, mediaNode): + if mediaNode["type"] == "image": + return self.serializeImageToProtobuf(mediaNode) + if mediaNode["type"] == "location": + return self.serializeLocationToProtobuf(mediaNode) + if mediaNode["type"] == "vcard": + return self.serializeContactToProtobuf(mediaNode) + + return None + + def serializeLocationToProtobuf(self, mediaNode): + m = Message() + location_message = LocationMessage() + location_message.degress_latitude = float(mediaNode["latitude"]) + location_message.degress_longitude = float(mediaNode["longitude"]) + location_message.address = mediaNode["name"] + location_message.name = mediaNode["name"] + location_message.url = mediaNode["url"] + + m.location_message.MergeFrom(location_message) + return m.SerializeToString() + + def serializeContactToProtobuf(self, mediaNode): + vcardNode = mediaNode.getChild("vcard") + m = Message() + contact_message = ContactMessage() + contact_message.display_name = vcardNode["name"] + m.vcard = vcardNode.getData() + m.contact_message.MergeFrom(contact_message) + + return m.SerializeToString() + + def serializeImageToProtobuf(self, mediaNode): + m = Message() + image_message = ImageMessage() + image_message.url = mediaNode["url"] + image_message.width = int(mediaNode["width"]) + image_message.height = int(mediaNode["height"]) + image_message.mime_type = mediaNode["mimetype"] + image_message.file_sha256 = mediaNode["filehash"] + image_message.file_length = int(mediaNode["size"]) + image_message.caption = mediaNode["caption"] or "" + image_message.jpeg_thumbnail = mediaNode.getData() + + m.image_message.MergeFrom(image_message) + return m.SerializeToString() + + def serializeUrlToProtobuf(self, node): + pass + + def serializeDocumentToProtobuf(self, node): pass ### keys set and get @@ -460,7 +527,6 @@ def onGetKeysError(self, errorNode, getKeysEntity): pass ### - def adjustArray(self, arr): return HexUtil.decodeHex(binascii.hexlify(arr)) diff --git a/yowsup/layers/axolotl/protocolentities/enc.py b/yowsup/layers/axolotl/protocolentities/enc.py index b181fa0a0..6d5655809 100644 --- a/yowsup/layers/axolotl/protocolentities/enc.py +++ b/yowsup/layers/axolotl/protocolentities/enc.py @@ -5,12 +5,13 @@ class EncProtocolEntity(ProtocolEntity): TYPE_MSG = "msg" TYPE_SKMSG = "skmsg" TYPES = (TYPE_PKMSG, TYPE_MSG, TYPE_SKMSG) - def __init__(self, type, version, data): + def __init__(self, type, version, data, mediaType = None): assert type in self.__class__.TYPES, "Unknown message enc type %s" % type super(EncProtocolEntity, self).__init__("enc") self.type = type self.version = int(version) self.data = data + self.mediaType = mediaType def getType(self): return self.type @@ -21,9 +22,15 @@ def getVersion(self): def getData(self): return self.data + def getMediaType(self): + return self.mediaType + def toProtocolTreeNode(self): - return ProtocolTreeNode("enc", {"type": self.type, "v": str(self.version)}, data = self.data) + attribs = {"type": self.type, "v": str(self.version)} + if self.mediaType: + attribs["mediatype"] = self.mediaType + return ProtocolTreeNode("enc", attribs, data = self.data) @staticmethod def fromProtocolTreeNode(node): - return EncProtocolEntity(node["type"], node["v"], node.data.encode('latin-1') if sys.version_info >= (3,0) else node.data) + return EncProtocolEntity(node["type"], node["v"], node.data.encode('latin-1') if sys.version_info >= (3,0) else node.data, node["mediatype"]) From 5b097194fd901f9efaef66411069a29cdf830c7f Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Wed, 20 Jan 2016 18:03:48 +0200 Subject: [PATCH 35/68] Allow autotrust identity via YowAxolotlLayer.PROP_IDENTITY_AUTOTRUST Fixes #1121 RROR:yowsup.layers.axolotl.layer:Untrusted identity!! Fixes #829 Fixes #1034 Untrusted identity exception Refs #1024 --- yowsup/demos/cli/stack.py | 2 ++ yowsup/layers/axolotl/layer.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/yowsup/demos/cli/stack.py b/yowsup/demos/cli/stack.py index 715110a8e..4994fcfe2 100644 --- a/yowsup/demos/cli/stack.py +++ b/yowsup/demos/cli/stack.py @@ -3,6 +3,7 @@ from yowsup.layers.auth import AuthError from yowsup.layers import YowLayerEvent from yowsup.layers.auth import YowAuthenticationProtocolLayer +from yowsup.layers.axolotl.layer import YowAxolotlLayer import sys class YowsupCliStack(object): @@ -16,6 +17,7 @@ def __init__(self, credentials, encryptionEnabled = True): # self.stack.setCredentials(credentials) self.stack.setCredentials(credentials) + self.stack.setProp(YowAxolotlLayer.PROP_IDENTITY_AUTOTRUST, True) def start(self): print("Yowsup Cli client\n==================\nType /help for available commands\n") diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index b4e4ac717..f437e33a4 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -37,6 +37,7 @@ class YowAxolotlLayer(YowProtocolLayer): EVENT_PREKEYS_SET = "org.openwhatsapp.yowsup.events.axololt.setkeys" + PROP_IDENTITY_AUTOTRUST = "org.openwhatsapp.yowsup.prop.axolotl.INDENTITY_AUTOTRUST" _STATE_INIT = 0 _STATE_GENKEYS = 1 _STATE_HASKEYS = 2 @@ -248,8 +249,13 @@ def handleEncMessage(self, node): self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"]).toProtocolTreeNode()) except UntrustedIdentityException as e: - logger.error(e) - logger.warning("Ignoring message with untrusted identity") + if(self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False)): + logger.warning("Autotrusting identity for %s" % e.getName()) + self.store.saveIdentity(e.getName(), e.getIdentityKey()) + return self.handleEncMessage(node) + else: + logger.error(e) + logger.warning("Ignoring message with untrusted identity") def handlePreKeyWhisperMessage(self, node): pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) From 0f181795bab2a65f3472159e74cbd07d908326f6 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 13 Feb 2016 19:53:40 +0100 Subject: [PATCH 36/68] New media send interface Added image media message builder Initial support for encrypted media (recv ok) Moved proto to protocol_messages --- yowsup/layers/axolotl/layer.py | 2 +- yowsup/layers/interface/interface.py | 97 ++++++++----------- .../builder_message_media_downloadable.py | 61 ++++++++++++ .../message_media_downloadable.py | 43 ++++---- .../message_media_downloadable_audio.py | 20 ++-- .../message_media_downloadable_image.py | 67 +++++++++---- .../message_media_downloadable_video.py | 7 +- .../protocol_messages/proto/__init__.py | 0 .../proto/wa.proto} | 0 .../proto/wa_pb2.py} | 48 ++++----- 10 files changed, 216 insertions(+), 129 deletions(-) create mode 100644 yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py create mode 100644 yowsup/layers/protocol_messages/proto/__init__.py rename yowsup/layers/{axolotl/proto/e2e.proto => protocol_messages/proto/wa.proto} (100%) rename yowsup/layers/{axolotl/e2e_pb2.py => protocol_messages/proto/wa_pb2.py} (88%) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index f437e33a4..826e2f2e2 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -3,7 +3,7 @@ from yowsup.layers.network.layer import YowNetworkLayer from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity -from yowsup.layers.axolotl.e2e_pb2 import * +from yowsup.layers.protocol_messages.proto.wa_pb2 import * from yowsup.layers.axolotl.store.sqlite.liteaxolotlstore import LiteAxolotlStore from yowsup.layers.axolotl.protocolentities import * from yowsup.structs import ProtocolTreeNode diff --git a/yowsup/layers/interface/interface.py b/yowsup/layers/interface/interface.py index 9674e37d1..763309e11 100644 --- a/yowsup/layers/interface/interface.py +++ b/yowsup/layers/interface/interface.py @@ -4,11 +4,12 @@ from yowsup.layers.auth import YowAuthenticationProtocolLayer from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity from yowsup.layers.protocol_acks.protocolentities import IncomingAckProtocolEntity -from yowsup.layers.network.layer import YowNetworkLayer -from yowsup.layers import EventCallback +from yowsup.layers.axolotl.layer import YowAxolotlLayer +from yowsup.layers.protocol_media.protocolentities.iq_requestupload import RequestUploadIqProtocolEntity +from yowsup.layers.protocol_media.protocolentities.iq_requestupload_result import ResultRequestUploadIqProtocolEntity +from yowsup.layers.protocol_media.mediauploader import MediaUploader +from yowsup.layers.axolotl.layer import YowAxolotlLayer import inspect -import logging -logger = logging.getLogger(__name__) class ProtocolEntityCallback(object): def __init__(self, entityType): @@ -17,15 +18,12 @@ def __init__(self, entityType): def __call__(self, fn): fn.entity_callback = self.entityType return fn - -class YowInterfaceLayer(YowLayer): - PROP_RECONNECT_ON_STREAM_ERR = "org.openwhatsapp.yowsup.prop.interface.reconnect_on_stream_error" +class YowInterfaceLayer(YowLayer): def __init__(self): super(YowInterfaceLayer, self).__init__() - self.reconnect = False self.entity_callbacks = {} self.iqRegistry = {} # self.receiptsRegistry = {} @@ -35,40 +33,13 @@ def __init__(self): fname = m[0] fn = m[1] self.entity_callbacks[fn.entity_callback] = getattr(self, fname) - + def _sendIq(self, iqEntity, onSuccess = None, onError = None): assert iqEntity.getTag() == "iq", "Expected *IqProtocolEntity in _sendIq, got %s" % iqEntity.getTag() self.iqRegistry[iqEntity.getId()] = (iqEntity, onSuccess, onError) self.toLower(iqEntity) - # def _sendReceipt(self, outgoingReceiptProtocolEntity, onAck = None): - # assert outgoingReceiptProtocolEntity.__class__ == OutgoingReceiptProtocolEntity,\ - # "Excepted OutgoingReceiptProtocolEntity in _sendReceipt, got %s" % outgoingReceiptProtocolEntity.__class__ - # self.receiptsRegistry[outgoingReceiptProtocolEntity.getId()] = (outgoingReceiptProtocolEntity, onAck) - # self.toLower(outgoingReceiptProtocolEntity) - - # def processReceiptsRegistry(self, incomingAckProtocolEntity): - # ''' - # entity: IncomingAckProtocolEntity - # ''' - # - # if incomingAckProtocolEntity.__class__ != IncomingAckProtocolEntity: - # return False - # - # receipt_id = incomingAckProtocolEntity.getId() - # if receipt_id in self.receiptsRegistry: - # originalReceiptEntity, ackClbk = self.receiptsRegistry[receipt_id] - # del self.receiptsRegistry[receipt_id] - # - # if ackClbk: - # ackClbk(incomingAckProtocolEntity, originalReceiptEntity) - # - # return True - # - # return False - - def processIqRegistry(self, entity): """ :type entity: IqProtocolEntity @@ -107,27 +78,41 @@ def receive(self, entity): self.entity_callbacks[entityType](entity) else: self.toUpper(entity) - - @ProtocolEntityCallback("stream:error") - def onStreamError(self, streamErrorEntity): - logger.error(streamErrorEntity) - if self.getProp(self.__class__.PROP_RECONNECT_ON_STREAM_ERR, True): - logger.info("Initiating reconnect") - self.reconnect = True - self.disconnect() + + def _sendMediaMessage(self, builder, success, error = None, progress = None): + # axolotlIface = self.getLayerInterface(YowAxolotlLayer) + # if axolotlIface: + # axolotlIface.encryptMedia(builder) + + iq = RequestUploadIqProtocolEntity(builder.mediaType, filePath = builder.getFilepath(), encrypted = builder.isEncrypted()) + successFn = lambda resultEntity, requestUploadEntity: self.__onRequestUploadSuccess(resultEntity, requestUploadEntity, builder, success, error, progress) + errorFn = lambda errorEntity, requestUploadEntity: self.__onRequestUploadError(errorEntity, requestUploadEntity, error) + self._sendIq(iq, successFn, errorFn) + + def __onRequestUploadSuccess(self, resultRequestUploadIqProtocolEntity, requestUploadEntity, builder, success, error = None, progress = None): + if(resultRequestUploadIqProtocolEntity.isDuplicate()): + return success(builder.build(resultRequestUploadIqProtocolEntity.getUrl(), resultRequestUploadIqProtocolEntity.getIp())) else: - logger.warn("No reconnecting because property %s is not set" % self.__class__.PROP_RECONNECT_ON_STREAM_ERR) - self.toUpper(streamErrorEntity) - - @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) - def onConnected(self, yowLayerEvent): - self.reconnect = False - - @EventCallback(YowNetworkLayer.EVENT_STATE_DISCONNECTED) - def onDisconnected(self, yowLayerEvent): - if self.reconnect: - self.reconnect = False - self.connect() + successFn = lambda path, jid, url: self.__onMediaUploadSuccess(builder, url, resultRequestUploadIqProtocolEntity.getIp(), success) + errorFn = lambda path, jid, errorText: self.__onMediaUploadError(builder, errorText, error) + + mediaUploader = MediaUploader(builder.jid, self.getOwnJid(), builder.getFilepath(), + resultRequestUploadIqProtocolEntity.getUrl(), + resultRequestUploadIqProtocolEntity.getResumeOffset(), + successFn, errorFn, progress, async=True) + mediaUploader.start() + + def __onRequestUploadError(self, errorEntity, requestUploadEntity, builder, error = None): + if error: + return error(errorEntity.code, errorEntity.text, errorEntity.backoff) + + def __onMediaUploadSuccess(self, builder, url, ip, successClbk): + messageNode = builder.build(url, ip) + return successClbk(messageNode) + + def __onMediaUploadError(self, builder, errorText, errorClbk = None): + if errorClbk: + return errorClbk(0, errorText, 0) def __str__(self): return "Interface Layer" diff --git a/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py b/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py new file mode 100644 index 000000000..458f7f25b --- /dev/null +++ b/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py @@ -0,0 +1,61 @@ +# from yowsup.layers.protocol_media import mediacipher +import tempfile +import os +class DownloadableMediaMessageBuilder(object): + def __init__(self, downloadbleMediaMessageClass, jid, filepath): + self.jid = jid + self.filepath = filepath + self.encryptedFilepath = None + self.cls = downloadbleMediaMessageClass + self.mediaKey = None + self.attributes = {} + self.mediaType = self.cls.__name__.split("DownloadableMediaMessageProtocolEntity")[0].lower() #ugly ? + + # def encrypt(self): + # fd, encpath = tempfile.mkstemp() + # mediaKey = os.urandom(112) + # keys = mediacipher.getDerivedKeys(mediaKey) + # out = mediacipher.encryptImage(self.filepath, keys) + # with open(encImagePath, 'w') as outF: + # outF.write(out) + # + # self.mediaKey = mediaKey + # self.encryptedFilepath = encpath + + # def decrypt(self): + # self.mediaKey = None + # self.encryptedFilePath = None + + + def setEncryptionData(self, mediaKey, encryptedFilepath): + self.mediaKey = mediaKey + self.encryptedFilepath = encryptedFilepath + + def isEncrypted(self): + return self.encryptedFilepath is not None + + def getFilepath(self): + return self.encryptedFilepath or self.filepath + + def getOriginalFilepath(self): + return self.filepath + + def set(self, key, val): + self.attributes[key] = val + + def get(self, key, default = None): + if key in self.attributes: + return self.attributes[key] + + return default + + def getOrSet(self, key, func): + if not self.get(key): + self.set(key, func()) + + def build(self, url = None, ip = None): + if url: + self.set("url", url) + if ip: + self.set("ip", ip) + return self.cls.fromBuilder(self) diff --git a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py index 0aaf4c64c..7286fdab9 100644 --- a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py +++ b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py @@ -4,27 +4,27 @@ import os class DownloadableMediaMessageProtocolEntity(MediaMessageProtocolEntity): ''' - {{THUMBNAIL_RAWDATA (JPEG?)}} ''' def __init__(self, mediaType, - mimeType, fileHash, url, ip, size, fileName, - _id = None, _from = None, to = None, notify = None, timestamp = None, + mimeType, fileHash, url, ip, size, fileName, mediaKey = None, + _id = None, _from = None, to = None, notify = None, timestamp = None, participant = None, preview = None, offline = None, retry = None): super(DownloadableMediaMessageProtocolEntity, self).__init__(mediaType, _id, _from, to, notify, timestamp, participant, preview, offline, retry) - self.setDownloadableMediaProps(mimeType, fileHash, url, ip, size, fileName) + self.setDownloadableMediaProps(mimeType, fileHash, url, ip, size, fileName, mediaKey) def __str__(self): out = super(DownloadableMediaMessageProtocolEntity, self).__str__() @@ -45,13 +45,14 @@ def getMediaUrl(self): def getMimeType(self): return self.mimeType - def setDownloadableMediaProps(self, mimeType, fileHash, url, ip, size, fileName): + def setDownloadableMediaProps(self, mimeType, fileHash, url, ip, size, fileName, mediaKey): self.mimeType = mimeType self.fileHash = fileHash self.url = url self.ip = ip self.size = int(size) self.fileName = fileName + self.mediaKey = mediaKey def toProtocolTreeNode(self): node = super(DownloadableMediaMessageProtocolEntity, self).toProtocolTreeNode() @@ -63,9 +64,14 @@ def toProtocolTreeNode(self): mediaNode.setAttribute("ip", self.ip) mediaNode.setAttribute("size", str(self.size)) mediaNode.setAttribute("file", self.fileName) + if self.mediaKey: + mediaNode.setAttribute("mediakey", self.mediaKey) return node + def isEncrypted(self): + return self.mediaKey is not None + @staticmethod def fromProtocolTreeNode(node): entity = MediaMessageProtocolEntity.fromProtocolTreeNode(node) @@ -77,17 +83,18 @@ def fromProtocolTreeNode(node): mediaNode.getAttributeValue("url"), mediaNode.getAttributeValue("ip"), mediaNode.getAttributeValue("size"), - mediaNode.getAttributeValue("file") + mediaNode.getAttributeValue("file"), + mediaNode.getAttributeValue("mediakey") ) return entity @staticmethod - def fromFilePath(fpath, url, mediaType, ip, to, mimeType = None, preview = None, filehash = None, filesize = None): - mimeType = mimeType or MimeTools.getMIME(fpath) - filehash = filehash or WATools.getFileHashForUpload(fpath) - size = filesize or os.path.getsize(fpath) - fileName = os.path.basename(fpath) - - return DownloadableMediaMessageProtocolEntity(mediaType, mimeType, filehash, url, ip, size, fileName, to = to, preview = preview) - - + def fromBuilder(builder): + url = builder.get("url") + ip = builder.get("ip") + assert url, "Url is required" + mimeType = builder.get("mimetype", MimeTools.getMIME(builder.getOriginalFilepath())[0]) + filehash = WATools.getFileHashForUpload(builder.getFilepath()) + size = os.path.getsize(builder.getFilepath()) + fileName = os.path.basename(builder.getFilepath()) + return DownloadableMediaMessageProtocolEntity(builder.mediaType, mimeType, filehash, url, ip, size, fileName, to = builder.jid, preview = builder.get("preview")) diff --git a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_audio.py b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_audio.py index f01aaf527..936681988 100644 --- a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_audio.py +++ b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_audio.py @@ -2,33 +2,33 @@ from .message_media_downloadable import DownloadableMediaMessageProtocolEntity class AudioDownloadableMediaMessageProtocolEntity(DownloadableMediaMessageProtocolEntity): ''' - {{THUMBNAIL_RAWDATA (JPEG?)}} ''' def __init__(self, - mimeType, fileHash, url, ip, size, fileName, + mimeType, fileHash, url, ip, size, fileName, abitrate, acodec, asampfreq, duration, encoding, origin, seconds, - _id = None, _from = None, to = None, notify = None, timestamp = None, + _id = None, _from = None, to = None, notify = None, timestamp = None, participant = None, preview = None, offline = None, retry = None): super(AudioDownloadableMediaMessageProtocolEntity, self).__init__("audio", - mimeType, fileHash, url, ip, size, fileName, + mimeType, fileHash, url, ip, size, fileName, None, _id, _from, to, notify, timestamp, participant, preview, offline, retry) self.setAudioProps(abitrate, acodec, asampfreq, duration, encoding, origin, seconds) diff --git a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py index 5c29526aa..2f1365569 100644 --- a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py +++ b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py @@ -1,35 +1,38 @@ from yowsup.structs import ProtocolEntity, ProtocolTreeNode from .message_media_downloadable import DownloadableMediaMessageProtocolEntity +from .builder_message_media_downloadable import DownloadableMediaMessageBuilder +from yowsup.layers.protocol_messages.proto.wa_pb2 import ImageMessage from yowsup.common.tools import ImageTools + class ImageDownloadableMediaMessageProtocolEntity(DownloadableMediaMessageProtocolEntity): ''' - {{THUMBNAIL_RAWDATA (JPEG?)}} ''' def __init__(self, mimeType, fileHash, url, ip, size, fileName, - encoding, width, height, caption = None, - _id = None, _from = None, to = None, notify = None, timestamp = None, + encoding, width, height, caption = None, mediaKey = None, + _id = None, _from = None, to = None, notify = None, timestamp = None, participant = None, preview = None, offline = None, retry = None): super(ImageDownloadableMediaMessageProtocolEntity, self).__init__("image", - mimeType, fileHash, url, ip, size, fileName, + mimeType, fileHash, url, ip, size, fileName, mediaKey, _id, _from, to, notify, timestamp, participant, preview, offline, retry) self.setImageProps(encoding, width, height, caption) @@ -50,7 +53,7 @@ def setImageProps(self, encoding, width, height, caption): def getCaption(self): return self.caption - + def toProtocolTreeNode(self): node = super(ImageDownloadableMediaMessageProtocolEntity, self).toProtocolTreeNode() mediaNode = node.getChild("media") @@ -63,6 +66,20 @@ def toProtocolTreeNode(self): return node + def toProtobufMessage(self): + image_message = ImageMessage() + image_message.url = self.url + image_message.width = self.width + image_message.height = self.height + image_message.mime_type = self.mimeType + image_message.file_sha256 = self.fileHash + image_message.file_length = self.size + image_message.caption = self.caption + image_message.jpeg_thumbnail = self.preview + image_message.media_key = self.mediaKey + + return image_message + @staticmethod def fromProtocolTreeNode(node): entity = DownloadableMediaMessageProtocolEntity.fromProtocolTreeNode(node) @@ -78,17 +95,29 @@ def fromProtocolTreeNode(node): @staticmethod - def fromFilePath(path, url, ip, to, mimeType = None, caption = None, dimensions = None): - preview = ImageTools.generatePreviewFromImage(path) - entity = DownloadableMediaMessageProtocolEntity.fromFilePath(path, url, DownloadableMediaMessageProtocolEntity.MEDIA_TYPE_IMAGE, ip, to, mimeType, preview) - entity.__class__ = ImageDownloadableMediaMessageProtocolEntity - - if not dimensions: - dimensions = ImageTools.getImageDimensions(path) + def getBuilder(jid, filepath): + return DownloadableMediaMessageBuilder(ImageDownloadableMediaMessageProtocolEntity, jid, filepath) + @staticmethod + def fromBuilder(builder): + builder.getOrSet("preview", lambda: ImageTools.generatePreviewFromImage(builder.getOriginalFilepath())) + filepath = builder.getFilepath() + caption = builder.get("caption") + dimensions = builder.get("dimensions", ImageTools.getImageDimensions(builder.getOriginalFilepath())) assert dimensions, "Could not determine image dimensions" - width, height = dimensions + + entity = DownloadableMediaMessageProtocolEntity.fromBuilder(builder) + entity.__class__ = builder.cls entity.setImageProps("raw", width, height, caption) return entity + @staticmethod + def fromFilePath(path, url, ip, to, mimeType = None, caption = None, dimensions = None): + builder = ImageDownloadableMediaMessageProtocolEntity.getBuilder(to, path) + builder.set("url", url) + builder.set("ip", ip) + builder.set("caption", caption) + builder.set("mimetype", mimeType) + builder.set("dimensions", dimesions) + return ImageDownloadableMediaMessageProtocolEntity.fromBuilder(builder) diff --git a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_video.py b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_video.py index c6850739c..c6acb3bfc 100644 --- a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_video.py +++ b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_video.py @@ -32,7 +32,7 @@ def __init__(self, participant = None, preview = None, offline = None, retry = None): super(VideoDownloadableMediaMessageProtocolEntity, self).__init__("video", - mimeType, fileHash, url, ip, size, fileName, + mimeType, fileHash, url, ip, size, fileName, None, _id, _from, to, notify, timestamp, participant, preview, offline, retry) self.setVideoProps(encoding, width, height, vbitrate, abitrate, acodec, asampfmt, asampfreq, duration, fps, seconds, vcodec, caption) @@ -75,6 +75,11 @@ def toProtocolTreeNode(self): node = super(VideoDownloadableMediaMessageProtocolEntity, self).toProtocolTreeNode() mediaNode = node.getChild("media") + mediaNode.setAttribute("abitrate", self.abitrate) + mediaNode.setAttribute("acodec", self.acodec) + mediaNode.setAttribute("asampfmt", self.asampfmt) + mediaNode.setAttribute("asampfreq", self.asampfreq) + mediaNode.setAttribute("duration", self.duration) mediaNode.setAttribute("encoding", self.encoding) mediaNode.setAttribute("height", str(self.height)) mediaNode.setAttribute("width", str(self.width)) diff --git a/yowsup/layers/protocol_messages/proto/__init__.py b/yowsup/layers/protocol_messages/proto/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yowsup/layers/axolotl/proto/e2e.proto b/yowsup/layers/protocol_messages/proto/wa.proto similarity index 100% rename from yowsup/layers/axolotl/proto/e2e.proto rename to yowsup/layers/protocol_messages/proto/wa.proto diff --git a/yowsup/layers/axolotl/e2e_pb2.py b/yowsup/layers/protocol_messages/proto/wa_pb2.py similarity index 88% rename from yowsup/layers/axolotl/e2e_pb2.py rename to yowsup/layers/protocol_messages/proto/wa_pb2.py index dbef983ba..528c1f77e 100644 --- a/yowsup/layers/axolotl/e2e_pb2.py +++ b/yowsup/layers/protocol_messages/proto/wa_pb2.py @@ -1,5 +1,5 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! -# source: e2e.proto +# source: wa.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) @@ -16,9 +16,9 @@ DESCRIPTOR = _descriptor.FileDescriptor( - name='e2e.proto', + name='wa.proto', package='com.whatsapp.proto', - serialized_pb=_b('\n\te2e.proto\x12\x12\x63om.whatsapp.proto\"\xa3\x03\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12Y\n\x1fsender_key_distribution_message\x18\x02 \x01(\x0b\x32\x30.com.whatsapp.proto.SenderKeyDistributionMessage\x12\x37\n\rimage_message\x18\x03 \x01(\x0b\x32 .com.whatsapp.proto.ImageMessage\x12;\n\x0f\x63ontact_message\x18\x04 \x01(\x0b\x32\".com.whatsapp.proto.ContactMessage\x12=\n\x10location_message\x18\x05 \x01(\x0b\x32#.com.whatsapp.proto.LocationMessage\x12=\n\x10\x64ocument_message\x18\x07 \x01(\x0b\x32#.com.whatsapp.proto.DocumentMessage\x12\x33\n\x0burl_message\x18\x06 \x01(\x0b\x32\x1e.com.whatsapp.proto.UrlMessage\"`\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x02(\t\x12/\n\'axolotl_sender_key_distribution_message\x18\x02 \x02(\x0c\"\xb3\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x02(\x0c\x12\x11\n\tmime_type\x18\x02 \x02(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x0e\n\x06height\x18\x06 \x02(\r\x12\r\n\x05width\x18\x07 \x02(\r\x12\x11\n\tmedia_key\x18\x08 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x8a\x01\n\x0fLocationMessage\x12\x18\n\x10\x64\x65grees_latitude\x18\x01 \x02(\x01\x12\x19\n\x11\x64\x65grees_longitude\x18\x02 \x02(\x01\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x02(\t\x12\x0b\n\x03url\x18\x05 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\xa8\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x02(\t\x12\x10\n\x08mimeType\x18\x02 \x02(\t\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x12\n\npage_count\x18\x06 \x02(\r\x12\x11\n\tmedia_key\x18\x07 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x83\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x14\n\x0cmatched_text\x18\x02 \x02(\t\x12\x15\n\rcanonical_url\x18\x04 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x02(\t\x12\r\n\x05title\x18\x06 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\t\"5\n\x0e\x43ontactMessage\x12\x14\n\x0c\x64isplay_name\x18\x01 \x02(\t\x12\r\n\x05vcard\x18\x10 \x02(\t') + serialized_pb=_b('\n\x08wa.proto\x12\x12\x63om.whatsapp.proto\"\xa3\x03\n\x07Message\x12\x14\n\x0c\x63onversation\x18\x01 \x01(\t\x12Y\n\x1fsender_key_distribution_message\x18\x02 \x01(\x0b\x32\x30.com.whatsapp.proto.SenderKeyDistributionMessage\x12\x37\n\rimage_message\x18\x03 \x01(\x0b\x32 .com.whatsapp.proto.ImageMessage\x12;\n\x0f\x63ontact_message\x18\x04 \x01(\x0b\x32\".com.whatsapp.proto.ContactMessage\x12=\n\x10location_message\x18\x05 \x01(\x0b\x32#.com.whatsapp.proto.LocationMessage\x12=\n\x10\x64ocument_message\x18\x07 \x01(\x0b\x32#.com.whatsapp.proto.DocumentMessage\x12\x33\n\x0burl_message\x18\x06 \x01(\x0b\x32\x1e.com.whatsapp.proto.UrlMessage\"`\n\x1cSenderKeyDistributionMessage\x12\x0f\n\x07groupId\x18\x01 \x02(\t\x12/\n\'axolotl_sender_key_distribution_message\x18\x02 \x02(\x0c\"\xb3\x01\n\x0cImageMessage\x12\x0b\n\x03url\x18\x01 \x02(\x0c\x12\x11\n\tmime_type\x18\x02 \x02(\t\x12\x0f\n\x07\x63\x61ption\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x0e\n\x06height\x18\x06 \x02(\r\x12\r\n\x05width\x18\x07 \x02(\r\x12\x11\n\tmedia_key\x18\x08 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x8a\x01\n\x0fLocationMessage\x12\x18\n\x10\x64\x65grees_latitude\x18\x01 \x02(\x01\x12\x19\n\x11\x64\x65grees_longitude\x18\x02 \x02(\x01\x12\x0c\n\x04name\x18\x03 \x02(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x04 \x02(\t\x12\x0b\n\x03url\x18\x05 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\xa8\x01\n\x0f\x44ocumentMessage\x12\x0b\n\x03url\x18\x01 \x02(\t\x12\x10\n\x08mimeType\x18\x02 \x02(\t\x12\r\n\x05title\x18\x03 \x02(\t\x12\x13\n\x0b\x66ile_sha256\x18\x04 \x02(\x0c\x12\x13\n\x0b\x66ile_length\x18\x05 \x02(\x04\x12\x12\n\npage_count\x18\x06 \x02(\r\x12\x11\n\tmedia_key\x18\x07 \x02(\x0c\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\x0c\"\x83\x01\n\nUrlMessage\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x14\n\x0cmatched_text\x18\x02 \x02(\t\x12\x15\n\rcanonical_url\x18\x04 \x02(\t\x12\x13\n\x0b\x64\x65scription\x18\x05 \x02(\t\x12\r\n\x05title\x18\x06 \x02(\t\x12\x16\n\x0ejpeg_thumbnail\x18\x10 \x02(\t\"5\n\x0e\x43ontactMessage\x12\x14\n\x0c\x64isplay_name\x18\x01 \x02(\t\x12\r\n\x05vcard\x18\x10 \x02(\t') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -92,8 +92,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=34, - serialized_end=453, + serialized_start=33, + serialized_end=452, ) @@ -129,8 +129,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=455, - serialized_end=551, + serialized_start=454, + serialized_end=550, ) @@ -215,8 +215,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=554, - serialized_end=733, + serialized_start=553, + serialized_end=732, ) @@ -280,8 +280,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=736, - serialized_end=874, + serialized_start=735, + serialized_end=873, ) @@ -359,8 +359,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=877, - serialized_end=1045, + serialized_start=876, + serialized_end=1044, ) @@ -424,8 +424,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1048, - serialized_end=1179, + serialized_start=1047, + serialized_end=1178, ) @@ -461,8 +461,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=1181, - serialized_end=1234, + serialized_start=1180, + serialized_end=1233, ) _MESSAGE.fields_by_name['sender_key_distribution_message'].message_type = _SENDERKEYDISTRIBUTIONMESSAGE @@ -481,49 +481,49 @@ Message = _reflection.GeneratedProtocolMessageType('Message', (_message.Message,), dict( DESCRIPTOR = _MESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.Message) )) _sym_db.RegisterMessage(Message) SenderKeyDistributionMessage = _reflection.GeneratedProtocolMessageType('SenderKeyDistributionMessage', (_message.Message,), dict( DESCRIPTOR = _SENDERKEYDISTRIBUTIONMESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.SenderKeyDistributionMessage) )) _sym_db.RegisterMessage(SenderKeyDistributionMessage) ImageMessage = _reflection.GeneratedProtocolMessageType('ImageMessage', (_message.Message,), dict( DESCRIPTOR = _IMAGEMESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.ImageMessage) )) _sym_db.RegisterMessage(ImageMessage) LocationMessage = _reflection.GeneratedProtocolMessageType('LocationMessage', (_message.Message,), dict( DESCRIPTOR = _LOCATIONMESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.LocationMessage) )) _sym_db.RegisterMessage(LocationMessage) DocumentMessage = _reflection.GeneratedProtocolMessageType('DocumentMessage', (_message.Message,), dict( DESCRIPTOR = _DOCUMENTMESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.DocumentMessage) )) _sym_db.RegisterMessage(DocumentMessage) UrlMessage = _reflection.GeneratedProtocolMessageType('UrlMessage', (_message.Message,), dict( DESCRIPTOR = _URLMESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.UrlMessage) )) _sym_db.RegisterMessage(UrlMessage) ContactMessage = _reflection.GeneratedProtocolMessageType('ContactMessage', (_message.Message,), dict( DESCRIPTOR = _CONTACTMESSAGE, - __module__ = 'e2e_pb2' + __module__ = 'wa_pb2' # @@protoc_insertion_point(class_scope:com.whatsapp.proto.ContactMessage) )) _sym_db.RegisterMessage(ContactMessage) From b5f287657798eb5e076233cc311e5c912704c321 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 13 Feb 2016 21:01:30 +0100 Subject: [PATCH 37/68] Still support v1 enc messages + fixed dupl image send Fixed builder not returning default when key exists but none --- yowsup/layers/axolotl/layer.py | 18 +++++++------- .../builder_message_media_downloadable.py | 2 +- .../message_media_downloadable_image.py | 2 +- yowsup/structs/protocoltreenode.py | 24 +++++++++---------- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index 826e2f2e2..77dd0c7f5 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -176,7 +176,7 @@ def processPendingIncomingMessages(self, jid): def handlePlaintextNode(self, node): recipient_id = node["to"].split('@')[0] - + v2 = node["to"] in self.v2Jids if not self.store.containsSession(recipient_id, 1): entity = GetKeysIqProtocolEntity([node["to"]]) if node["to"] not in self.pendingMessages: @@ -184,19 +184,18 @@ def handlePlaintextNode(self, node): self.pendingMessages[node["to"]].append(node) self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingMessages), self.onGetKeysError) - else: - message = None - messagebytes = self.serializeToProtobuf(node) - if messagebytes: + elif v2 or not node.getChild("media"): #media enc is only for v2 messsages + messageData = self.serializeToProtobuf(node) if v2 else node.getChild("body").getData() + if messageData: sessionCipher = self.getSessionCipher(recipient_id) - ciphertext = sessionCipher.encrypt(messagebytes) + ciphertext = sessionCipher.encrypt(messageData) mediaType = node.getChild("media")["type"] if node.getChild("media") else None encEntity = EncryptedMessageProtocolEntity( [ EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , - 2, + 2 if v2 else 1, ciphertext.serialize(), mediaType)], "text" if not mediaType else "media", _id= node["id"], @@ -210,6 +209,9 @@ def handlePlaintextNode(self, node): self.toLower(encEntity.toProtocolTreeNode()) else: #case of unserializable messages (audio, video) ? self.toLower(node) + else: + self.toLower(node) + def handleEncMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) @@ -281,7 +283,7 @@ def handleWhisperMessage(self, node): padding = ord(plaintext[-1]) & 0xFF self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) else: - self.handleConversationMessage(encMessageProtocolEntity, plaintext) + self.handleConversationMessage(encMessageProtocolEntity.toProtocolTreeNode(), plaintext) def handleSenderKeyMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) diff --git a/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py b/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py index 458f7f25b..c60930234 100644 --- a/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py +++ b/yowsup/layers/protocol_media/protocolentities/builder_message_media_downloadable.py @@ -44,7 +44,7 @@ def set(self, key, val): self.attributes[key] = val def get(self, key, default = None): - if key in self.attributes: + if key in self.attributes and self.attributes[key] is not None: return self.attributes[key] return default diff --git a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py index 2f1365569..cc7b2e5f6 100644 --- a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py +++ b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable_image.py @@ -119,5 +119,5 @@ def fromFilePath(path, url, ip, to, mimeType = None, caption = None, dimensions builder.set("ip", ip) builder.set("caption", caption) builder.set("mimetype", mimeType) - builder.set("dimensions", dimesions) + builder.set("dimensions", dimensions) return ImageDownloadableMediaMessageProtocolEntity.fromBuilder(builder) diff --git a/yowsup/structs/protocoltreenode.py b/yowsup/structs/protocoltreenode.py index 139758ad3..2a4847598 100644 --- a/yowsup/structs/protocoltreenode.py +++ b/yowsup/structs/protocoltreenode.py @@ -51,7 +51,7 @@ def toString(self): if self.attributes is not None: for key,val in self.attributes.items(): if val is None: - raise Exception("None val for key: "+key); + raise ValueError("value is none for attr %s" % key) out+= " "+key+'="'+val+'"' out+= ">\n" @@ -74,7 +74,7 @@ def toString(self): out += "\nHEX3:%s\n" % binascii.hexlify(self.data.encode('latin-1')) else: out += "\nHEX:%s\n" % binascii.hexlify(self.data) - + for c in self.children: try: out += c.toString() @@ -83,27 +83,27 @@ def toString(self): out+= "\n" return out - + def __str__(self): - return self.toString() + return self.toString() def getData(self): return self.data def setData(self, data): self.data = data - - - @staticmethod + + + @staticmethod def tagEquals(node,string): return node is not None and node.tag is not None and node.tag == string - - + + @staticmethod def require(node,string): if not ProtocolTreeNode.tagEquals(node,string): raise Exception("failed require. string: "+string); - + def __getitem__(self, key): return self.getAttributeValue(key) @@ -138,7 +138,7 @@ def addChild(self, childNode): def addChildren(self, children): for c in children: self.addChild(c) - + def getAttributeValue(self,string): try: return self.attributes[string] @@ -156,7 +156,7 @@ def getAllChildren(self,tag = None): ret = [] if tag is None: return self.children - + for c in self.children: if tag == c.tag: ret.append(c) From 79c1515d31a272fa41199a759958a6688d21ba2e Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Mon, 15 Feb 2016 06:59:56 +0100 Subject: [PATCH 38/68] Fixed group detect and sender key dist msg handling --- yowsup/layers/axolotl/layer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py index 77dd0c7f5..c0c73a4c8 100644 --- a/yowsup/layers/axolotl/layer.py +++ b/yowsup/layers/axolotl/layer.py @@ -215,7 +215,7 @@ def handlePlaintextNode(self, node): def handleEncMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - isGroup = node["from"] if node["participant"] is None else node["from"] + isGroup = node["participant"] is not None senderJid = node["participant"] if isGroup else node["from"] encNode = None if node.getChild("enc")["v"] == "2" and senderJid not in self.v2Jids: @@ -318,7 +318,7 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): if m.HasField("sender_key_distribution_message"): axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) - if m.HasField("conversation"): + elif m.HasField("conversation"): self.handleConversationMessage(node, m.conversation) elif m.HasField("contact_message"): self.handleContactMessage(node, m.contact_message) From e6315bbb15aa3f14c5139b57aa94127c819e0216 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 1 Apr 2016 15:03:22 +0200 Subject: [PATCH 39/68] Fixed env get in auth layer --- yowsup/layers/auth/layer_authentication.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yowsup/layers/auth/layer_authentication.py b/yowsup/layers/auth/layer_authentication.py index 8962d73b4..1183391f5 100644 --- a/yowsup/layers/auth/layer_authentication.py +++ b/yowsup/layers/auth/layer_authentication.py @@ -7,8 +7,8 @@ from .autherror import AuthError from .protocolentities import * from yowsup.common.tools import StorageTools +from yowsup.env import YowsupEnv from .layer_interface_authentication import YowAuthenticationProtocolLayerInterface -from yowsup.env import CURRENT_ENV import base64 class YowAuthenticationProtocolLayer(YowProtocolLayer): @@ -123,6 +123,7 @@ def _sendResponse(self,nonce): def generateAuthBlob(self, nonce): keys = KeyStream.generateKeys(self.credentials[1], nonce) + currentEnv = YowsupEnv.getCurrent() inputKey = KeyStream(keys[2], keys[3]) outputKey = KeyStream(keys[0], keys[1]) @@ -144,10 +145,10 @@ def generateAuthBlob(self, nonce): nums.extend(time_bytes) strCat = "\x00\x00\x00\x00\x00\x00\x00\x00" - strCat += CURRENT_ENV.getOSVersion() + "\x00" - strCat += CURRENT_ENV.getManufacturer() + "\x00" - strCat += CURRENT_ENV.getDeviceName() + "\x00" - strCat += CURRENT_ENV.getBuildVersion() + strCat += currentEnv.getOSVersion() + "\x00" + strCat += currentEnv.getManufacturer() + "\x00" + strCat += currentEnv.getDeviceName() + "\x00" + strCat += currentEnv.getBuildVersion() nums.extend(list(map(ord, strCat))) encoded = outputKey.encodeMessage(nums, 0, 4, len(nums) - 4) From 29ba81babf44eb8b011146283a2ae55df314b21c Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 1 Apr 2016 23:23:21 +0200 Subject: [PATCH 40/68] Fixed readPacked8, closes #1431 --- yowsup/layers/coder/decoder.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index afd6bd296..e6ffa4f46 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -70,17 +70,18 @@ def readNibble(self, data): def readPacked8(self, n, data): size = self.readInt8(data) remove = 0 - if (size & 0x80) != 0: + if (size & 0x80) != 0 and n == 251: remove = 1 - size = size & 0x7F + size = size & 0x7F text = bytearray(self.readArray(size, data)) hexData = binascii.hexlify(text).upper() + dataSize = len(hexData) out = [] if remove == 0: - for i in range(0, len(hexData)): + for i in range(0, dataSize): char = chr(hexData[i]) if type(hexData[i]) is int else hexData[i] #python2/3 compat val = ord(binascii.unhexlify("0%s" % char)) - if i == (size - 1) and val > 11 and n != 251: continue + if i == (dataSize - 1) and val > 11 and n != 251: continue out.append(self.unpackByte(n, val)) else: out = map(ord, list(hexData[0: -remove])) if sys.version_info < (3,0) else list(hexData[0: -remove]) @@ -136,7 +137,7 @@ def readInt20(self, data): int1 = data.pop(0) int2 = data.pop(0) int3 = data.pop(0) - return ((int1 & 0xF) << 16) + (int2 << 8) + (int3 << 0) + return ((int1 & 0xF) << 16) | (int2 << 8) | int3 def readInt24(self, data): int1 = data.pop(0) From 8cb3a84dc100ad7d688650d3db9e7ebafaddc3a7 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 9 Apr 2016 16:15:23 +0200 Subject: [PATCH 41/68] Split up axolotl into 3 layers --- yowsup/demos/cli/stack.py | 4 +- yowsup/layers/axolotl/__init__.py | 5 +- yowsup/layers/axolotl/layer_base.py | 34 ++++ yowsup/layers/axolotl/layer_control.py | 136 +++++++++++++ yowsup/layers/axolotl/layer_receive.py | 269 +++++++++++++++++++++++++ yowsup/layers/axolotl/layer_send.py | 197 ++++++++++++++++++ yowsup/stacks/yowstack.py | 5 +- 7 files changed, 645 insertions(+), 5 deletions(-) create mode 100644 yowsup/layers/axolotl/layer_base.py create mode 100644 yowsup/layers/axolotl/layer_control.py create mode 100644 yowsup/layers/axolotl/layer_receive.py create mode 100644 yowsup/layers/axolotl/layer_send.py diff --git a/yowsup/demos/cli/stack.py b/yowsup/demos/cli/stack.py index 4994fcfe2..fa5c84fdc 100644 --- a/yowsup/demos/cli/stack.py +++ b/yowsup/demos/cli/stack.py @@ -3,7 +3,7 @@ from yowsup.layers.auth import AuthError from yowsup.layers import YowLayerEvent from yowsup.layers.auth import YowAuthenticationProtocolLayer -from yowsup.layers.axolotl.layer import YowAxolotlLayer +from yowsup.layers.axolotl import AxolotlReceivelayer import sys class YowsupCliStack(object): @@ -17,7 +17,7 @@ def __init__(self, credentials, encryptionEnabled = True): # self.stack.setCredentials(credentials) self.stack.setCredentials(credentials) - self.stack.setProp(YowAxolotlLayer.PROP_IDENTITY_AUTOTRUST, True) + self.stack.setProp(AxolotlReceivelayer.PROP_IDENTITY_AUTOTRUST, True) def start(self): print("Yowsup Cli client\n==================\nType /help for available commands\n") diff --git a/yowsup/layers/axolotl/__init__.py b/yowsup/layers/axolotl/__init__.py index 46422dc4b..3b53b7894 100644 --- a/yowsup/layers/axolotl/__init__.py +++ b/yowsup/layers/axolotl/__init__.py @@ -1 +1,4 @@ -from .layer import YowAxolotlLayer \ No newline at end of file +# from .layer import YowAxolotlLayer +from .layer_send import AxolotlSendLayer +from .layer_control import AxolotlControlLayer +from .layer_receive import AxolotlReceivelayer diff --git a/yowsup/layers/axolotl/layer_base.py b/yowsup/layers/axolotl/layer_base.py new file mode 100644 index 000000000..6f6b2341f --- /dev/null +++ b/yowsup/layers/axolotl/layer_base.py @@ -0,0 +1,34 @@ +from yowsup.layers.axolotl.store.sqlite.liteaxolotlstore import LiteAxolotlStore +from yowsup.layers import YowProtocolLayer, YowLayerEvent, EventCallback +from yowsup.common.tools import StorageTools +from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer + + +class AxolotlBaseLayer(YowProtocolLayer): + _DB = "axolotl.db" + def __init__(self): + super(AxolotlBaseLayer, self).__init__() + self.store = None + + def onNewStoreSet(self, store): + pass + + def send(self, node): + self.toLower(node) + + @property + def store(self): + if self._store is None: + self.store = LiteAxolotlStore( + StorageTools.constructPath( + self.getProp( + YowAuthenticationProtocolLayer.PROP_CREDENTIALS)[0], + self.__class__._DB + ) + ) + return self._store + + @store.setter + def store(self, store): + self._store = store + self.onNewStoreSet(self._store) diff --git a/yowsup/layers/axolotl/layer_control.py b/yowsup/layers/axolotl/layer_control.py new file mode 100644 index 000000000..e2e1a9236 --- /dev/null +++ b/yowsup/layers/axolotl/layer_control.py @@ -0,0 +1,136 @@ +from .layer_base import AxolotlBaseLayer +from yowsup.layers import YowLayerEvent, EventCallback +from yowsup.layers.network.layer import YowNetworkLayer +from axolotl.util.keyhelper import KeyHelper +from yowsup.layers.axolotl.protocolentities import * +from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer +from axolotl.util.hexutil import HexUtil +from axolotl.ecc.curve import Curve +import logging +import binascii +import sys + +logger = logging.getLogger(__name__) + +class AxolotlControlLayer(AxolotlBaseLayer): + _STATE_INIT = 0 + _STATE_GENKEYS = 1 + _STATE_HASKEYS = 2 + _COUNT_PREKEYS = 200 + EVENT_PREKEYS_SET = "org.openwhatsapp.yowsup.events.axololt.setkeys" + + def __init__(self): + super(AxolotlControlLayer, self).__init__() + self.state = self.__class__._STATE_INIT + + def onNewStoreSet(self, store): + super(AxolotlControlLayer, self).onNewStoreSet(store) + if store is not None: + self.state = self.__class__._STATE_HASKEYS if store.getLocalRegistrationId() is not None \ + else self.__class__._STATE_INIT + + def receive(self, protocolTreeNode): + """ + :type protocolTreeNode: ProtocolTreeNode + """ + if not self.processIqRegistry(protocolTreeNode): + if protocolTreeNode.tag == "notification" and protocolTreeNode["type"] == "encrypt": + self.onEncryptNotification(protocolTreeNode) + return + self.toUpper(protocolTreeNode) + + def isInitState(self): + return self.store == None or self.state == self.__class__._STATE_INIT + + def isGenKeysState(self): + return self.state == self.__class__._STATE_GENKEYS + + + def onEncryptNotification(self, protocolTreeNode): + entity = EncryptNotification.fromProtocolTreeNode(protocolTreeNode) + ack = OutgoingAckProtocolEntity(protocolTreeNode["id"], "notification", protocolTreeNode["type"], protocolTreeNode["from"]) + self.toLower(ack.toProtocolTreeNode()) + self.sendKeys(fresh=False, countPreKeys = self.__class__._COUNT_PREKEYS - entity.getCount()) + + @EventCallback(EVENT_PREKEYS_SET) + def onPreKeysSet(self, yowLayerEvent): + self.sendKeys(fresh=False) + + @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) + def onConnected(self, yowLayerEvent): + if self.isInitState(): + self.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, True) + + @EventCallback(YowAuthenticationProtocolLayer.EVENT_AUTHED) + def onAuthed(self, yowLayerEvent): + if yowLayerEvent.getArg("passive") and self.isInitState(): + logger.info("Axolotl layer is generating keys") + self.sendKeys() + + @EventCallback(YowNetworkLayer.EVENT_STATE_DISCONNECTED) + def onDisconnected(self, yowLayerEvent): + if self.isGenKeysState(): + #we requested this disconnect in this layer to switch off passive + #no need to traverse it to upper layers? + self.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, False) + self.state = self.__class__._STATE_HASKEYS + self.getLayerInterface(YowNetworkLayer).connect() + else: + self.store = None + ### keys set and get + def sendKeys(self, fresh = True, countPreKeys = _COUNT_PREKEYS): + identityKeyPair = KeyHelper.generateIdentityKeyPair() if fresh else self.store.getIdentityKeyPair() + registrationId = KeyHelper.generateRegistrationId() if fresh else self.store.getLocalRegistrationId() + preKeys = KeyHelper.generatePreKeys(KeyHelper.getRandomSequence(), countPreKeys) + signedPreKey = KeyHelper.generateSignedPreKey(identityKeyPair, KeyHelper.getRandomSequence(65536)) + preKeysDict = {} + for preKey in preKeys: + keyPair = preKey.getKeyPair() + preKeysDict[self.adjustId(preKey.getId())] = self.adjustArray(keyPair.getPublicKey().serialize()[1:]) + + signedKeyTuple = (self.adjustId(signedPreKey.getId()), + self.adjustArray(signedPreKey.getKeyPair().getPublicKey().serialize()[1:]), + self.adjustArray(signedPreKey.getSignature())) + + setKeysIq = SetKeysIqProtocolEntity(self.adjustArray(identityKeyPair.getPublicKey().serialize()[1:]), signedKeyTuple, preKeysDict, Curve.DJB_TYPE, self.adjustId(registrationId)) + + onResult = lambda _, __: self.persistKeys(registrationId, identityKeyPair, preKeys, signedPreKey, fresh) + self._sendIq(setKeysIq, onResult, self.onSentKeysError) + + def persistKeys(self, registrationId, identityKeyPair, preKeys, signedPreKey, fresh): + total = len(preKeys) + curr = 0 + prevPercentage = 0 + + if fresh: + self.store.storeLocalData(registrationId, identityKeyPair) + self.store.storeSignedPreKey(signedPreKey.getId(), signedPreKey) + + for preKey in preKeys: + self.store.storePreKey(preKey.getId(), preKey) + curr += 1 + currPercentage = int((curr * 100) / total) + if currPercentage == prevPercentage: + continue + prevPercentage = currPercentage + #logger.debug("%s" % currPercentage + "%") + sys.stdout.write("Storing prekeys %d%% \r" % (currPercentage)) + sys.stdout.flush() + + if fresh: + self.state = self.__class__._STATE_GENKEYS + self.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_DISCONNECT)) + + def onSentKeysError(self, errorNode, keysEntity): + raise Exception("Sent keys were not accepted") + + def adjustArray(self, arr): + return HexUtil.decodeHex(binascii.hexlify(arr)) + + def adjustId(self, _id): + _id = format(_id, 'x') + zfiller = len(_id) if len(_id) % 2 == 0 else len(_id) + 1 + _id = _id.zfill(zfiller if zfiller > 6 else 6) + # if len(_id) % 2: + # _id = "0" + _id + return binascii.unhexlify(_id) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py new file mode 100644 index 000000000..d3f2c2ef5 --- /dev/null +++ b/yowsup/layers/axolotl/layer_receive.py @@ -0,0 +1,269 @@ +from .layer_base import AxolotlBaseLayer + +from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity +from yowsup.layers.protocol_messages.proto.wa_pb2 import * +from yowsup.layers.axolotl.protocolentities import * +from yowsup.structs import ProtocolTreeNode + +from axolotl.protocol.prekeywhispermessage import PreKeyWhisperMessage +from axolotl.protocol.whispermessage import WhisperMessage +from axolotl.sessioncipher import SessionCipher +from axolotl.groups.groupcipher import GroupCipher +from axolotl.invalidmessageexception import InvalidMessageException +from axolotl.duplicatemessagexception import DuplicateMessageException +from axolotl.invalidkeyidexception import InvalidKeyIdException +from axolotl.nosessionexception import NoSessionException +from axolotl.untrustedidentityexception import UntrustedIdentityException +from axolotl.axolotladdress import AxolotlAddress +from axolotl.groups.senderkeyname import SenderKeyName +from axolotl.groups.groupsessionbuilder import GroupSessionBuilder +from axolotl.protocol.senderkeydistributionmessage import SenderKeyDistributionMessage + +import logging +import copy +logger = logging.getLogger(__name__) + +class AxolotlReceivelayer(AxolotlBaseLayer): + PROP_IDENTITY_AUTOTRUST = "org.openwhatsapp.yowsup.prop.axolotl.INDENTITY_AUTOTRUST" + def __init__(self): + super(AxolotlReceivelayer, self).__init__() + self.v2Jids = [] #people we're going to send v2 enc messages + self.sessionCiphers = {} + self.groupCiphers = {} + self.pendingIncomingMessages = {} + + def receive(self, protocolTreeNode): + """ + :type protocolTreeNode: ProtocolTreeNode + """ + if not self.processIqRegistry(protocolTreeNode): + if protocolTreeNode.tag == "message": + self.onMessage(protocolTreeNode) + return + elif protocolTreeNode.tag == "receipt" and protocolTreeNode["type"] == "retry": + # should bring up that message, resend it, but in upper layer? + # as it might have to be fetched from a persistent storage + pass + self.toUpper(protocolTreeNode) + + + def processPendingIncomingMessages(self, jid): + if jid in self.pendingIncomingMessages: + for messageNode in self.pendingIncomingMessages[jid]: + self.onMessage(messageNode) + + del self.pendingIncomingMessages[jid] + + ##### handling received data ##### + + def onMessage(self, protocolTreeNode): + encNode = protocolTreeNode.getChild("enc") + if encNode: + self.handleEncMessage(protocolTreeNode) + else: + self.toUpper(protocolTreeNode) + + def handleEncMessage(self, node): + encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) + isGroup = node["participant"] is not None + senderJid = node["participant"] if isGroup else node["from"] + encNode = None + if node.getChild("enc")["v"] == "2" and senderJid not in self.v2Jids: + self.v2Jids.append(senderJid) + try: + if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG): + self.handlePreKeyWhisperMessage(node) + elif encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG): + self.handleWhisperMessage(node) + if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG): + self.handleSenderKeyMessage(node) + except InvalidMessageException as e: + logger.error(e) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) + retry.setRegData(self.store.getLocalRegistrationId()) + self.toLower(retry.toProtocolTreeNode()) + except InvalidKeyIdException as e: + logger.error(e) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) + retry.setRegData(self.store.getLocalRegistrationId()) + self.toLower(retry.toProtocolTreeNode()) + except NoSessionException as e: + logger.error(e) + entity = GetKeysIqProtocolEntity([senderJid]) + if senderJid not in self.pendingIncomingMessages: + self.pendingIncomingMessages[senderJid] = [] + self.pendingIncomingMessages[senderJid].append(node) + + self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingIncomingMessages), self.onGetKeysError) + except DuplicateMessageException as e: + logger.error(e) + logger.warning("Going to send the delivery receipt myself !") + self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"]).toProtocolTreeNode()) + + except UntrustedIdentityException as e: + if(self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False)): + logger.warning("Autotrusting identity for %s" % e.getName()) + self.store.saveIdentity(e.getName(), e.getIdentityKey()) + return self.handleEncMessage(node) + else: + logger.error(e) + logger.warning("Ignoring message with untrusted identity") + def handlePreKeyWhisperMessage(self, node): + pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) + enc = pkMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG) + preKeyWhisperMessage = PreKeyWhisperMessage(serialized=enc.getData()) + sessionCipher = self.getSessionCipher(pkMessageProtocolEntity.getAuthor(False)) + plaintext = sessionCipher.decryptPkmsg(preKeyWhisperMessage) + if enc.getVersion() == 2: + padding = ord(plaintext[-1]) & 0xFF + self.parseAndHandleMessageProto(pkMessageProtocolEntity, plaintext[:-padding]) + else: + self.handleConversationMessage(node, plaintext) + + def handleWhisperMessage(self, node): + encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) + + enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG) + whisperMessage = WhisperMessage(serialized=enc.getData()) + sessionCipher = self.getSessionCipher(encMessageProtocolEntity.getAuthor(False)) + plaintext = sessionCipher.decryptMsg(whisperMessage) + + if enc.getVersion() == 2: + padding = ord(plaintext[-1]) & 0xFF + self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) + else: + self.handleConversationMessage(encMessageProtocolEntity.toProtocolTreeNode(), plaintext) + + def handleSenderKeyMessage(self, node): + encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) + enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG) + + senderKeyName = SenderKeyName(encMessageProtocolEntity.getFrom(True), AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0)) + groupCipher = GroupCipher(self.store, senderKeyName) + try: + plaintext = groupCipher.decrypt(enc.getData()) + padding = ord(plaintext[-1]) & 0xFF + self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) + except NoSessionException as e: + logger.error(e) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) + retry.setRegData(self.store.getLocalRegistrationId()) + self.toLower(retry.toProtocolTreeNode()) + + + def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): + node = encMessageProtocolEntity.toProtocolTreeNode() + m = Message() + try: + m.ParseFromString(serializedData) + except: + print("DUMP:") + print(serializedData) + print([s for s in serializedData]) + print([ord(s) for s in serializedData]) + raise + if not m or not serializedData: + raise ValueError("Empty message") + + if m.HasField("sender_key_distribution_message"): + axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) + self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) + elif m.HasField("conversation"): + self.handleConversationMessage(node, m.conversation) + elif m.HasField("contact_message"): + self.handleContactMessage(node, m.contact_message) + elif m.HasField("url_message"): + self.handleUrlMessage(node, m.url_message) + elif m.HasField("location_message"): + self.handleLocationMessage(node, m.location_message) + elif m.HasField("image_message"): + self.handleImageMessage(node, m.image_message) + else: + print(m) + raise ValueError("Unhandled") + + def handleSenderKeyDistributionMessage(self, senderKeyDistributionMessage, axolotlAddress): + groupId = senderKeyDistributionMessage.groupId + axolotlSenderKeyDistributionMessage = SenderKeyDistributionMessage(serialized=senderKeyDistributionMessage.axolotl_sender_key_distribution_message) + groupSessionBuilder = GroupSessionBuilder(self.store) + senderKeyName = SenderKeyName(groupId, axolotlAddress) + groupSessionBuilder.process(senderKeyName, axolotlSenderKeyDistributionMessage) + + def handleConversationMessage(self, originalEncNode, text): + messageNode = copy.deepcopy(originalEncNode) + messageNode.children = [] + messageNode.addChild(ProtocolTreeNode("body", data = text)) + self.toUpper(messageNode) + + def handleImageMessage(self, originalEncNode, imageMessage): + messageNode = copy.deepcopy(originalEncNode) + messageNode["type"] = "media" + mediaNode = ProtocolTreeNode("media", { + "type": "image", + "filehash": imageMessage.file_sha256, + "size": str(imageMessage.file_length), + "url": imageMessage.url, + "mimetype": imageMessage.mime_type, + "width": imageMessage.width, + "height": imageMessage.height, + "caption": imageMessage.caption, + "encoding": "raw", + "file": "enc", + "ip": "0" + }, data = imageMessage.jpeg_thumbnail) + messageNode.addChild(mediaNode) + + self.toUpper(messageNode) + + def handleUrlMessage(self, originalEncNode, urlMessage): + #convert to ?? + pass + + def handleDocumentMessage(self, originalEncNode, documentMessage): + #convert to ?? + pass + + def handleLocationMessage(self, originalEncNode, locationMessage): + messageNode = copy.deepycopy(originalEncNode) + messageNode["type"] = "media" + mediaNode = ProtocolTreeNode("media", { + "latitude": locationMessage.degrees_latitude, + "longitude": locationMessage.degress_longitude, + "name": "%s %s" % (locationMessage.name, locationMessage.address), + "url": locationMessage.url, + "encoding": "raw", + "type": "location" + }, data=locationMessage.jpeg_thumbnail) + messageNode.addChild(mediaNode) + self.toUpper(messageNode) + + def handleContactMessage(self, originalEncNode, contactMessage): + messageNode = copy.deepycopy(originalEncNode) + messageNode["type"] = "media" + mediaNode = ProtocolTreeNode("media", { + "type": "vcard" + }, [ + ProtocolTreeNode("vcard", {"name": contactMessage.display_name}, data = contactMessage.vcard) + ] ) + messageNode.addChild(mediaNode) + self.toUpper(messageNode) + + + + def getSessionCipher(self, recipientId): + if recipientId in self.sessionCiphers: + sessionCipher = self.sessionCiphers[recipientId] + else: + sessionCipher = SessionCipher(self.store, self.store, self.store, self.store, recipientId, 1) + self.sessionCiphers[recipientId] = sessionCipher + + return sessionCipher + + def getGroupCipher(self, groupId, senderId): + senderKeyName = SenderKeyName(groupId, AxolotlAddress(senderId, 1)) + if senderKeyName in self.groupCiphers: + groupCipher = self.groupCiphers[senderKeyName] + else: + groupCipher = GroupCipher(self.store, senderKeyName) + self.groupCiphers[senderKeyName] = groupCipher + return groupCipher diff --git a/yowsup/layers/axolotl/layer_send.py b/yowsup/layers/axolotl/layer_send.py new file mode 100644 index 000000000..1e8653fe3 --- /dev/null +++ b/yowsup/layers/axolotl/layer_send.py @@ -0,0 +1,197 @@ +from yowsup.layers.protocol_messages.proto.wa_pb2 import * +from yowsup.layers.axolotl.protocolentities import * + +from axolotl.sessionbuilder import SessionBuilder +from axolotl.protocol.whispermessage import WhisperMessage +from axolotl.sessioncipher import SessionCipher +from axolotl.groups.groupcipher import GroupCipher +from axolotl.axolotladdress import AxolotlAddress +from axolotl.groups.senderkeyname import SenderKeyName + +import copy +import logging + +from .layer_base import AxolotlBaseLayer + +logger = logging.getLogger(__name__) + +class AxolotlSendLayer(AxolotlBaseLayer): + def __init__(self): + super(AxolotlSendLayer, self).__init__() + + self.sessionCiphers = {} + self.groupCiphers = {} + self.pendingMessages = {} + self.skipEncJids = [] + self.v2Jids = [] + + def __str__(self): + return "Axolotl Layer" + + + def send(self, node): + if node.tag == "message" and node["to"] not in self.skipEncJids: + self.handlePlaintextNode(node) + return + self.toLower(node) + + + def processPendingMessages(self, jid): + if jid in self.pendingMessages: + for messageNode in self.pendingMessages[jid]: + if jid in self.skipEncJids: + self.toLower(messageNode) + else: + self.handlePlaintextNode(messageNode) + + del self.pendingMessages[jid] + + + + #### handling message types + + def handlePlaintextNode(self, node): + recipient_id = node["to"].split('@')[0] + v2 = node["to"] in self.v2Jids + if not self.store.containsSession(recipient_id, 1): + entity = GetKeysIqProtocolEntity([node["to"]]) + if node["to"] not in self.pendingMessages: + self.pendingMessages[node["to"]] = [] + self.pendingMessages[node["to"]].append(node) + + self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingMessages), self.onGetKeysError) + elif v2 or not node.getChild("media"): #media enc is only for v2 messsages + messageData = self.serializeToProtobuf(node) if v2 else node.getChild("body").getData() + if messageData: + sessionCipher = self.getSessionCipher(recipient_id) + ciphertext = sessionCipher.encrypt(messageData) + + mediaType = node.getChild("media")["type"] if node.getChild("media") else None + + encEntity = EncryptedMessageProtocolEntity( + [ + EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , + 2 if v2 else 1, + ciphertext.serialize(), mediaType)], + "text" if not mediaType else "media", + _id= node["id"], + to = node["to"], + notify = node["notify"], + timestamp= node["timestamp"], + participant=node["participant"], + offline=node["offline"], + retry=node["retry"] + ) + self.toLower(encEntity.toProtocolTreeNode()) + else: #case of unserializable messages (audio, video) ? + self.toLower(node) + else: + self.toLower(node) + + def serializeToProtobuf(self, node): + if node.getChild("body"): + return self.serializeTextToProtobuf(node) + elif node.getChild("media"): + return self.serializeMediaToProtobuf(node.getChild("media")) + else: + raise ValueError("No body or media nodes found") + + def serializeTextToProtobuf(self, node): + m = Message() + m.conversation = node.getChild("body").getData() + return m.SerializeToString() + + def serializeMediaToProtobuf(self, mediaNode): + if mediaNode["type"] == "image": + return self.serializeImageToProtobuf(mediaNode) + if mediaNode["type"] == "location": + return self.serializeLocationToProtobuf(mediaNode) + if mediaNode["type"] == "vcard": + return self.serializeContactToProtobuf(mediaNode) + + return None + + def serializeLocationToProtobuf(self, mediaNode): + m = Message() + location_message = LocationMessage() + location_message.degress_latitude = float(mediaNode["latitude"]) + location_message.degress_longitude = float(mediaNode["longitude"]) + location_message.address = mediaNode["name"] + location_message.name = mediaNode["name"] + location_message.url = mediaNode["url"] + + m.location_message.MergeFrom(location_message) + return m.SerializeToString() + + def serializeContactToProtobuf(self, mediaNode): + vcardNode = mediaNode.getChild("vcard") + m = Message() + contact_message = ContactMessage() + contact_message.display_name = vcardNode["name"] + m.vcard = vcardNode.getData() + m.contact_message.MergeFrom(contact_message) + + return m.SerializeToString() + + def serializeImageToProtobuf(self, mediaNode): + m = Message() + image_message = ImageMessage() + image_message.url = mediaNode["url"] + image_message.width = int(mediaNode["width"]) + image_message.height = int(mediaNode["height"]) + image_message.mime_type = mediaNode["mimetype"] + image_message.file_sha256 = mediaNode["filehash"] + image_message.file_length = int(mediaNode["size"]) + image_message.caption = mediaNode["caption"] or "" + image_message.jpeg_thumbnail = mediaNode.getData() + + m.image_message.MergeFrom(image_message) + return m.SerializeToString() + + def serializeUrlToProtobuf(self, node): + pass + + def serializeDocumentToProtobuf(self, node): + pass + + def onGetKeysResult(self, resultNode, getKeysEntity, processPendingFn): + entity = ResultGetKeysIqProtocolEntity.fromProtocolTreeNode(resultNode) + + resultJids = entity.getJids() + for jid in getKeysEntity.getJids(): + + if jid not in resultJids: + self.skipEncJids.append(jid) + self.processPendingMessages(jid) + continue + + recipient_id = jid.split('@')[0] + preKeyBundle = entity.getPreKeyBundleFor(jid) + + sessionBuilder = SessionBuilder(self.store, self.store, self.store, + self.store, recipient_id, 1) + sessionBuilder.processPreKeyBundle(preKeyBundle) + + processPendingFn(jid) + + def onGetKeysError(self, errorNode, getKeysEntity): + pass + ### + + def getSessionCipher(self, recipientId): + if recipientId in self.sessionCiphers: + sessionCipher = self.sessionCiphers[recipientId] + else: + sessionCipher = SessionCipher(self.store, self.store, self.store, self.store, recipientId, 1) + self.sessionCiphers[recipientId] = sessionCipher + + return sessionCipher + + def getGroupCipher(self, groupId, senderId): + senderKeyName = SenderKeyName(groupId, AxolotlAddress(senderId, 1)) + if senderKeyName in self.groupCiphers: + groupCipher = self.groupCiphers[senderKeyName] + else: + groupCipher = GroupCipher(self.store, senderKeyName) + self.groupCiphers[senderKeyName] = groupCipher + return groupCipher diff --git a/yowsup/stacks/yowstack.py b/yowsup/stacks/yowstack.py index 81236e7dc..9142a53e2 100644 --- a/yowsup/stacks/yowstack.py +++ b/yowsup/stacks/yowstack.py @@ -70,8 +70,9 @@ def getDefaultLayers(axolotl = False, groups = True, media = True, privacy = Tru allLayers = coreLayers if axolotl: - from yowsup.layers.axolotl import YowAxolotlLayer - allLayers += (YowAxolotlLayer,) + from yowsup.layers.axolotl import AxolotlSendLayer, AxolotlControlLayer, AxolotlReceivelayer + allLayers += (AxolotlControlLayer,) + allLayers += (YowParallelLayer((AxolotlSendLayer, AxolotlReceivelayer)),) allLayers += (YowParallelLayer(protocolLayers),) From 1325f49d3dd86edf3395967a4296b9f966f1891d Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 9 Apr 2016 16:24:02 +0200 Subject: [PATCH 42/68] Don't send/receive in Base layer, fixed participant receipt --- yowsup/layers/axolotl/layer_base.py | 5 ++++- yowsup/layers/axolotl/layer_control.py | 3 +++ yowsup/layers/axolotl/layer_receive.py | 4 +++- yowsup/layers/axolotl/layer_send.py | 1 - 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/yowsup/layers/axolotl/layer_base.py b/yowsup/layers/axolotl/layer_base.py index 6f6b2341f..efc19191b 100644 --- a/yowsup/layers/axolotl/layer_base.py +++ b/yowsup/layers/axolotl/layer_base.py @@ -14,7 +14,10 @@ def onNewStoreSet(self, store): pass def send(self, node): - self.toLower(node) + pass + + def receive(self, node): + pass @property def store(self): diff --git a/yowsup/layers/axolotl/layer_control.py b/yowsup/layers/axolotl/layer_control.py index e2e1a9236..04cec6b7f 100644 --- a/yowsup/layers/axolotl/layer_control.py +++ b/yowsup/layers/axolotl/layer_control.py @@ -29,6 +29,9 @@ def onNewStoreSet(self, store): self.state = self.__class__._STATE_HASKEYS if store.getLocalRegistrationId() is not None \ else self.__class__._STATE_INIT + def send(self, node): + self.toLower(node) + def receive(self, protocolTreeNode): """ :type protocolTreeNode: ProtocolTreeNode diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index d3f2c2ef5..254c7a6b2 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -32,6 +32,8 @@ def __init__(self): self.groupCiphers = {} self.pendingIncomingMessages = {} + + def receive(self, protocolTreeNode): """ :type protocolTreeNode: ProtocolTreeNode @@ -98,7 +100,7 @@ def handleEncMessage(self, node): except DuplicateMessageException as e: logger.error(e) logger.warning("Going to send the delivery receipt myself !") - self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"]).toProtocolTreeNode()) + self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"], participant=node["participant"]).toProtocolTreeNode()) except UntrustedIdentityException as e: if(self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False)): diff --git a/yowsup/layers/axolotl/layer_send.py b/yowsup/layers/axolotl/layer_send.py index 1e8653fe3..bc2cd2698 100644 --- a/yowsup/layers/axolotl/layer_send.py +++ b/yowsup/layers/axolotl/layer_send.py @@ -8,7 +8,6 @@ from axolotl.axolotladdress import AxolotlAddress from axolotl.groups.senderkeyname import SenderKeyName -import copy import logging from .layer_base import AxolotlBaseLayer From 59c561cb4297f131fe37bc9bbbea1d48c86739c5 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 10 Apr 2016 23:54:27 +0200 Subject: [PATCH 43/68] Initial working group enc send and receive --- yowsup/layers/axolotl/layer.py | 565 ------------------ yowsup/layers/axolotl/layer_base.py | 31 +- yowsup/layers/axolotl/layer_receive.py | 37 +- yowsup/layers/axolotl/layer_send.py | 261 +++++--- yowsup/layers/axolotl/protocolentities/enc.py | 12 +- .../protocolentities/message_encrypted.py | 10 +- 6 files changed, 254 insertions(+), 662 deletions(-) delete mode 100644 yowsup/layers/axolotl/layer.py diff --git a/yowsup/layers/axolotl/layer.py b/yowsup/layers/axolotl/layer.py deleted file mode 100644 index c0c73a4c8..000000000 --- a/yowsup/layers/axolotl/layer.py +++ /dev/null @@ -1,565 +0,0 @@ -from yowsup.layers import YowProtocolLayer, YowLayerEvent, EventCallback -from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity -from yowsup.layers.network.layer import YowNetworkLayer -from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer -from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity -from yowsup.layers.protocol_messages.proto.wa_pb2 import * -from yowsup.layers.axolotl.store.sqlite.liteaxolotlstore import LiteAxolotlStore -from yowsup.layers.axolotl.protocolentities import * -from yowsup.structs import ProtocolTreeNode -from yowsup.common.tools import StorageTools - -from axolotl.sessionbuilder import SessionBuilder -from axolotl.util.keyhelper import KeyHelper -from axolotl.ecc.curve import Curve -from axolotl.protocol.prekeywhispermessage import PreKeyWhisperMessage -from axolotl.protocol.whispermessage import WhisperMessage -from axolotl.protocol.senderkeymessage import SenderKeyMessage -from axolotl.sessioncipher import SessionCipher -from axolotl.groups.groupcipher import GroupCipher -from axolotl.util.hexutil import HexUtil -from axolotl.invalidmessageexception import InvalidMessageException -from axolotl.duplicatemessagexception import DuplicateMessageException -from axolotl.invalidkeyidexception import InvalidKeyIdException -from axolotl.nosessionexception import NoSessionException -from axolotl.untrustedidentityexception import UntrustedIdentityException -from axolotl.axolotladdress import AxolotlAddress -from axolotl.groups.senderkeyname import SenderKeyName -from axolotl.groups.groupsessionbuilder import GroupSessionBuilder -from axolotl.protocol.senderkeydistributionmessage import SenderKeyDistributionMessage - -import binascii -import copy -import sys -import logging - -logger = logging.getLogger(__name__) - -class YowAxolotlLayer(YowProtocolLayer): - EVENT_PREKEYS_SET = "org.openwhatsapp.yowsup.events.axololt.setkeys" - PROP_IDENTITY_AUTOTRUST = "org.openwhatsapp.yowsup.prop.axolotl.INDENTITY_AUTOTRUST" - _STATE_INIT = 0 - _STATE_GENKEYS = 1 - _STATE_HASKEYS = 2 - _COUNT_PREKEYS = 200 - _DB = "axolotl.db" - - def __init__(self): - super(YowAxolotlLayer, self).__init__() - self.store = None - self.state = self.__class__._STATE_INIT - - self.sessionCiphers = {} - self.groupCiphers = {} - self.pendingMessages = {} - self.pendingIncomingMessages = {} - self.skipEncJids = [] - self.v2Jids = [] #people we're going to send v2 enc messages - - @property - def store(self): - if self._store is None: - self.store = LiteAxolotlStore( - StorageTools.constructPath( - self.getProp( - YowAuthenticationProtocolLayer.PROP_CREDENTIALS)[0], - self.__class__._DB - ) - ) - self.state = self.__class__._STATE_HASKEYS if self.store.getLocalRegistrationId() is not None \ - else self.__class__._STATE_INIT - - return self._store - - @store.setter - def store(self, store): - self._store = store - - def __str__(self): - return "Axolotl Layer" - - ### store and state - - - def isInitState(self): - return self.store == None or self.state == self.__class__._STATE_INIT - - def isGenKeysState(self): - return self.state == self.__class__._STATE_GENKEYS - ######## - - - @EventCallback(EVENT_PREKEYS_SET) - def onPreKeysSet(self, yowLayerEvent): - self.sendKeys(fresh=False) - - @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) - def onConnected(self, yowLayerEvent): - if self.isInitState(): - self.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, True) - - @EventCallback(YowAuthenticationProtocolLayer.EVENT_AUTHED) - def onAuthed(self, yowLayerEvent): - if yowLayerEvent.getArg("passive") and self.isInitState(): - logger.info("Axolotl layer is generating keys") - self.sendKeys() - - @EventCallback(YowNetworkLayer.EVENT_STATE_DISCONNECTED) - def onDisconnected(self, yowLayerEvent): - if self.isGenKeysState(): - #we requested this disconnect in this layer to switch off passive - #no need to traverse it to upper layers? - self.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, False) - self.state = self.__class__._STATE_HASKEYS - self.getLayerInterface(YowNetworkLayer).connect() - else: - self.store = None - - def send(self, node): - if node.tag == "message" and node["to"] not in self.skipEncJids: - self.handlePlaintextNode(node) - return - self.toLower(node) - - def receive(self, protocolTreeNode): - """ - :type protocolTreeNode: ProtocolTreeNode - """ - if not self.processIqRegistry(protocolTreeNode): - if protocolTreeNode.tag == "message": - self.onMessage(protocolTreeNode) - return - elif protocolTreeNode.tag == "notification" and protocolTreeNode["type"] == "encrypt": - self.onEncryptNotification(protocolTreeNode) - return - elif protocolTreeNode.tag == "receipt" and protocolTreeNode["type"] == "retry": - # should bring up that message, resend it, but in upper layer? - # as it might have to be fetched from a persistent storage - pass - self.toUpper(protocolTreeNode) - ###### - - ##### handling received data ##### - def onEncryptNotification(self, protocolTreeNode): - entity = EncryptNotification.fromProtocolTreeNode(protocolTreeNode) - ack = OutgoingAckProtocolEntity(protocolTreeNode["id"], "notification", protocolTreeNode["type"], protocolTreeNode["from"]) - self.toLower(ack.toProtocolTreeNode()) - self.sendKeys(fresh=False, countPreKeys = self.__class__._COUNT_PREKEYS - entity.getCount()) - - def onMessage(self, protocolTreeNode): - encNode = protocolTreeNode.getChild("enc") - if encNode: - self.handleEncMessage(protocolTreeNode) - else: - self.toUpper(protocolTreeNode) - - #### - - def processPendingMessages(self, jid): - if jid in self.pendingMessages: - for messageNode in self.pendingMessages[jid]: - if jid in self.skipEncJids: - self.toLower(messageNode) - else: - self.handlePlaintextNode(messageNode) - - del self.pendingMessages[jid] - - def processPendingIncomingMessages(self, jid): - if jid in self.pendingIncomingMessages: - for messageNode in self.pendingIncomingMessages[jid]: - self.onMessage(messageNode) - - del self.pendingIncomingMessages[jid] - - #### handling message types - - def handlePlaintextNode(self, node): - recipient_id = node["to"].split('@')[0] - v2 = node["to"] in self.v2Jids - if not self.store.containsSession(recipient_id, 1): - entity = GetKeysIqProtocolEntity([node["to"]]) - if node["to"] not in self.pendingMessages: - self.pendingMessages[node["to"]] = [] - self.pendingMessages[node["to"]].append(node) - - self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingMessages), self.onGetKeysError) - elif v2 or not node.getChild("media"): #media enc is only for v2 messsages - messageData = self.serializeToProtobuf(node) if v2 else node.getChild("body").getData() - if messageData: - sessionCipher = self.getSessionCipher(recipient_id) - ciphertext = sessionCipher.encrypt(messageData) - - mediaType = node.getChild("media")["type"] if node.getChild("media") else None - - encEntity = EncryptedMessageProtocolEntity( - [ - EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , - 2 if v2 else 1, - ciphertext.serialize(), mediaType)], - "text" if not mediaType else "media", - _id= node["id"], - to = node["to"], - notify = node["notify"], - timestamp= node["timestamp"], - participant=node["participant"], - offline=node["offline"], - retry=node["retry"] - ) - self.toLower(encEntity.toProtocolTreeNode()) - else: #case of unserializable messages (audio, video) ? - self.toLower(node) - else: - self.toLower(node) - - - def handleEncMessage(self, node): - encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - isGroup = node["participant"] is not None - senderJid = node["participant"] if isGroup else node["from"] - encNode = None - if node.getChild("enc")["v"] == "2" and senderJid not in self.v2Jids: - self.v2Jids.append(senderJid) - try: - if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG): - self.handlePreKeyWhisperMessage(node) - elif encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG): - self.handleWhisperMessage(node) - if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG): - self.handleSenderKeyMessage(node) - except InvalidMessageException as e: - logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) - retry.setRegData(self.store.getLocalRegistrationId()) - self.toLower(retry.toProtocolTreeNode()) - except InvalidKeyIdException as e: - logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) - retry.setRegData(self.store.getLocalRegistrationId()) - self.toLower(retry.toProtocolTreeNode()) - except NoSessionException as e: - logger.error(e) - entity = GetKeysIqProtocolEntity([senderJid]) - if senderJid not in self.pendingIncomingMessages: - self.pendingIncomingMessages[senderJid] = [] - self.pendingIncomingMessages[senderJid].append(node) - - self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingIncomingMessages), self.onGetKeysError) - except DuplicateMessageException as e: - logger.error(e) - logger.warning("Going to send the delivery receipt myself !") - self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"]).toProtocolTreeNode()) - - except UntrustedIdentityException as e: - if(self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False)): - logger.warning("Autotrusting identity for %s" % e.getName()) - self.store.saveIdentity(e.getName(), e.getIdentityKey()) - return self.handleEncMessage(node) - else: - logger.error(e) - logger.warning("Ignoring message with untrusted identity") - - def handlePreKeyWhisperMessage(self, node): - pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - enc = pkMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG) - preKeyWhisperMessage = PreKeyWhisperMessage(serialized=enc.getData()) - sessionCipher = self.getSessionCipher(pkMessageProtocolEntity.getAuthor(False)) - plaintext = sessionCipher.decryptPkmsg(preKeyWhisperMessage) - if enc.getVersion() == 2: - padding = ord(plaintext[-1]) & 0xFF - self.parseAndHandleMessageProto(pkMessageProtocolEntity, plaintext[:-padding]) - else: - self.handleConversationMessage(node, plaintext) - - def handleWhisperMessage(self, node): - encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - - enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_MSG) - whisperMessage = WhisperMessage(serialized=enc.getData()) - sessionCipher = self.getSessionCipher(encMessageProtocolEntity.getAuthor(False)) - plaintext = sessionCipher.decryptMsg(whisperMessage) - - if enc.getVersion() == 2: - padding = ord(plaintext[-1]) & 0xFF - self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) - else: - self.handleConversationMessage(encMessageProtocolEntity.toProtocolTreeNode(), plaintext) - - def handleSenderKeyMessage(self, node): - encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) - enc = encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG) - - senderKeyName = SenderKeyName(encMessageProtocolEntity.getFrom(True), AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0)) - groupCipher = GroupCipher(self.store, senderKeyName) - try: - plaintext = groupCipher.decrypt(enc.getData()) - padding = ord(plaintext[-1]) & 0xFF - self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) - except NoSessionException as e: - logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) - retry.setRegData(self.store.getLocalRegistrationId()) - self.toLower(retry.toProtocolTreeNode()) - - def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): - node = encMessageProtocolEntity.toProtocolTreeNode() - m = Message() - try: - m.ParseFromString(serializedData) - except: - print("DUMP:") - print(serializedData) - print([s for s in serializedData]) - print([ord(s) for s in serializedData]) - raise - if not m or not serializedData: - raise ValueError("Empty message") - - if m.HasField("sender_key_distribution_message"): - axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) - self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) - elif m.HasField("conversation"): - self.handleConversationMessage(node, m.conversation) - elif m.HasField("contact_message"): - self.handleContactMessage(node, m.contact_message) - elif m.HasField("url_message"): - self.handleUrlMessage(node, m.url_message) - elif m.HasField("location_message"): - self.handleLocationMessage(node, m.location_message) - elif m.HasField("image_message"): - self.handleImageMessage(node, m.image_message) - else: - print(m) - raise ValueError("Unhandled") - - def handleSenderKeyDistributionMessage(self, senderKeyDistributionMessage, axolotlAddress): - groupId = senderKeyDistributionMessage.groupId - axolotlSenderKeyDistributionMessage = SenderKeyDistributionMessage(serialized=senderKeyDistributionMessage.axolotl_sender_key_distribution_message) - groupSessionBuilder = GroupSessionBuilder(self.store) - senderKeyName = SenderKeyName(groupId, axolotlAddress) - groupSessionBuilder.process(senderKeyName, axolotlSenderKeyDistributionMessage) - - def handleConversationMessage(self, originalEncNode, text): - messageNode = copy.deepcopy(originalEncNode) - messageNode.children = [] - messageNode.addChild(ProtocolTreeNode("body", data = text)) - self.toUpper(messageNode) - - def handleImageMessage(self, originalEncNode, imageMessage): - messageNode = copy.deepcopy(originalEncNode) - messageNode["type"] = "media" - mediaNode = ProtocolTreeNode("media", { - "type": "image", - "filehash": imageMessage.file_sha256, - "size": str(imageMessage.file_length), - "url": imageMessage.url, - "mimetype": imageMessage.mime_type, - "width": imageMessage.width, - "height": imageMessage.height, - "caption": imageMessage.caption, - "encoding": "raw", - "file": "enc", - "ip": "0" - }, data = imageMessage.jpeg_thumbnail) - messageNode.addChild(mediaNode) - - self.toUpper(messageNode) - - def handleUrlMessage(self, originalEncNode, urlMessage): - #convert to ?? - pass - - def handleDocumentMessage(self, originalEncNode, documentMessage): - #convert to ?? - pass - - def handleLocationMessage(self, originalEncNode, locationMessage): - messageNode = copy.deepycopy(originalEncNode) - messageNode["type"] = "media" - mediaNode = ProtocolTreeNode("media", { - "latitude": locationMessage.degrees_latitude, - "longitude": locationMessage.degress_longitude, - "name": "%s %s" % (locationMessage.name, locationMessage.address), - "url": locationMessage.url, - "encoding": "raw", - "type": "location" - }, data=locationMessage.jpeg_thumbnail) - messageNode.addChild(mediaNode) - self.toUpper(messageNode) - - def handleContactMessage(self, originalEncNode, contactMessage): - messageNode = copy.deepycopy(originalEncNode) - messageNode["type"] = "media" - mediaNode = ProtocolTreeNode("media", { - "type": "vcard" - }, [ - ProtocolTreeNode("vcard", {"name": contactMessage.display_name}, data = contactMessage.vcard) - ] ) - messageNode.addChild(mediaNode) - self.toUpper(messageNode) - - def serializeToProtobuf(self, node): - if node.getChild("body"): - return self.serializeTextToProtobuf(node) - elif node.getChild("media"): - return self.serializeMediaToProtobuf(node.getChild("media")) - else: - raise ValueError("No body or media nodes found") - - def serializeTextToProtobuf(self, node): - m = Message() - m.conversation = node.getChild("body").getData() - return m.SerializeToString() - - def serializeMediaToProtobuf(self, mediaNode): - if mediaNode["type"] == "image": - return self.serializeImageToProtobuf(mediaNode) - if mediaNode["type"] == "location": - return self.serializeLocationToProtobuf(mediaNode) - if mediaNode["type"] == "vcard": - return self.serializeContactToProtobuf(mediaNode) - - return None - - def serializeLocationToProtobuf(self, mediaNode): - m = Message() - location_message = LocationMessage() - location_message.degress_latitude = float(mediaNode["latitude"]) - location_message.degress_longitude = float(mediaNode["longitude"]) - location_message.address = mediaNode["name"] - location_message.name = mediaNode["name"] - location_message.url = mediaNode["url"] - - m.location_message.MergeFrom(location_message) - return m.SerializeToString() - - def serializeContactToProtobuf(self, mediaNode): - vcardNode = mediaNode.getChild("vcard") - m = Message() - contact_message = ContactMessage() - contact_message.display_name = vcardNode["name"] - m.vcard = vcardNode.getData() - m.contact_message.MergeFrom(contact_message) - - return m.SerializeToString() - - def serializeImageToProtobuf(self, mediaNode): - m = Message() - image_message = ImageMessage() - image_message.url = mediaNode["url"] - image_message.width = int(mediaNode["width"]) - image_message.height = int(mediaNode["height"]) - image_message.mime_type = mediaNode["mimetype"] - image_message.file_sha256 = mediaNode["filehash"] - image_message.file_length = int(mediaNode["size"]) - image_message.caption = mediaNode["caption"] or "" - image_message.jpeg_thumbnail = mediaNode.getData() - - m.image_message.MergeFrom(image_message) - return m.SerializeToString() - - def serializeUrlToProtobuf(self, node): - pass - - def serializeDocumentToProtobuf(self, node): - pass - - ### keys set and get - def sendKeys(self, fresh = True, countPreKeys = _COUNT_PREKEYS): - identityKeyPair = KeyHelper.generateIdentityKeyPair() if fresh else self.store.getIdentityKeyPair() - registrationId = KeyHelper.generateRegistrationId() if fresh else self.store.getLocalRegistrationId() - preKeys = KeyHelper.generatePreKeys(KeyHelper.getRandomSequence(), countPreKeys) - signedPreKey = KeyHelper.generateSignedPreKey(identityKeyPair, KeyHelper.getRandomSequence(65536)) - preKeysDict = {} - for preKey in preKeys: - keyPair = preKey.getKeyPair() - preKeysDict[self.adjustId(preKey.getId())] = self.adjustArray(keyPair.getPublicKey().serialize()[1:]) - - signedKeyTuple = (self.adjustId(signedPreKey.getId()), - self.adjustArray(signedPreKey.getKeyPair().getPublicKey().serialize()[1:]), - self.adjustArray(signedPreKey.getSignature())) - - setKeysIq = SetKeysIqProtocolEntity(self.adjustArray(identityKeyPair.getPublicKey().serialize()[1:]), signedKeyTuple, preKeysDict, Curve.DJB_TYPE, self.adjustId(registrationId)) - - onResult = lambda _, __: self.persistKeys(registrationId, identityKeyPair, preKeys, signedPreKey, fresh) - self._sendIq(setKeysIq, onResult, self.onSentKeysError) - - def persistKeys(self, registrationId, identityKeyPair, preKeys, signedPreKey, fresh): - total = len(preKeys) - curr = 0 - prevPercentage = 0 - - if fresh: - self.store.storeLocalData(registrationId, identityKeyPair) - self.store.storeSignedPreKey(signedPreKey.getId(), signedPreKey) - - for preKey in preKeys: - self.store.storePreKey(preKey.getId(), preKey) - curr += 1 - currPercentage = int((curr * 100) / total) - if currPercentage == prevPercentage: - continue - prevPercentage = currPercentage - #logger.debug("%s" % currPercentage + "%") - sys.stdout.write("Storing prekeys %d%% \r" % (currPercentage)) - sys.stdout.flush() - - if fresh: - self.state = self.__class__._STATE_GENKEYS - self.broadcastEvent(YowLayerEvent(YowNetworkLayer.EVENT_STATE_DISCONNECT)) - - def onSentKeysError(self, errorNode, keysEntity): - raise Exception("Sent keys were not accepted") - - def onGetKeysResult(self, resultNode, getKeysEntity, processPendingFn): - entity = ResultGetKeysIqProtocolEntity.fromProtocolTreeNode(resultNode) - - resultJids = entity.getJids() - for jid in getKeysEntity.getJids(): - - if jid not in resultJids: - self.skipEncJids.append(jid) - self.processPendingMessages(jid) - continue - - recipient_id = jid.split('@')[0] - preKeyBundle = entity.getPreKeyBundleFor(jid) - - sessionBuilder = SessionBuilder(self.store, self.store, self.store, - self.store, recipient_id, 1) - sessionBuilder.processPreKeyBundle(preKeyBundle) - - processPendingFn(jid) - - def onGetKeysError(self, errorNode, getKeysEntity): - pass - ### - - def adjustArray(self, arr): - return HexUtil.decodeHex(binascii.hexlify(arr)) - - def adjustId(self, _id): - _id = format(_id, 'x') - zfiller = len(_id) if len(_id) % 2 == 0 else len(_id) + 1 - _id = _id.zfill(zfiller if zfiller > 6 else 6) - # if len(_id) % 2: - # _id = "0" + _id - return binascii.unhexlify(_id) - - def getSessionCipher(self, recipientId): - if recipientId in self.sessionCiphers: - sessionCipher = self.sessionCiphers[recipientId] - else: - sessionCipher = SessionCipher(self.store, self.store, self.store, self.store, recipientId, 1) - self.sessionCiphers[recipientId] = sessionCipher - - return sessionCipher - - def getGroupCipher(self, groupId, senderId): - senderKeyName = SenderKeyName(groupId, AxolotlAddress(senderId, 1)) - if senderKeyName in self.groupCiphers: - groupCipher = self.groupCiphers[senderKeyName] - else: - groupCipher = GroupCipher(self.store, senderKeyName) - self.groupCiphers[senderKeyName] = groupCipher - return groupCipher diff --git a/yowsup/layers/axolotl/layer_base.py b/yowsup/layers/axolotl/layer_base.py index efc19191b..aa9629253 100644 --- a/yowsup/layers/axolotl/layer_base.py +++ b/yowsup/layers/axolotl/layer_base.py @@ -1,14 +1,17 @@ from yowsup.layers.axolotl.store.sqlite.liteaxolotlstore import LiteAxolotlStore -from yowsup.layers import YowProtocolLayer, YowLayerEvent, EventCallback +from yowsup.layers import YowProtocolLayer from yowsup.common.tools import StorageTools from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer +from yowsup.layers.axolotl.protocolentities import * +from axolotl.sessionbuilder import SessionBuilder class AxolotlBaseLayer(YowProtocolLayer): _DB = "axolotl.db" def __init__(self): super(AxolotlBaseLayer, self).__init__() self.store = None + self.skipEncJids = [] def onNewStoreSet(self, store): pass @@ -17,7 +20,7 @@ def send(self, node): pass def receive(self, node): - pass + self.processIqRegistry(node) @property def store(self): @@ -35,3 +38,27 @@ def store(self): def store(self, store): self._store = store self.onNewStoreSet(self._store) + + def getKeysFor(self, jids, successClbk, errorClbk=None): + def onSuccess(resultNode, getKeysEntity): + entity = ResultGetKeysIqProtocolEntity.fromProtocolTreeNode(resultNode) + resultJids = entity.getJids() + + for jid in getKeysEntity.getJids(): + if jid not in resultJids: + self.skipEncJids.append(jid) + continue + + recipient_id = jid.split('@')[0] + preKeyBundle = entity.getPreKeyBundleFor(jid) + sessionBuilder = SessionBuilder(self.store, self.store, self.store, self.store, recipient_id, 1) + sessionBuilder.processPreKeyBundle(preKeyBundle) + + successClbk() + + def onError(errorNode, getKeysEntity): + if errorClbk: + errorClbk() + + entity = GetKeysIqProtocolEntity(jids) + self._sendIq(entity, onSuccess, onError=onError) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index 254c7a6b2..cf3fc44f9 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -32,8 +32,6 @@ def __init__(self): self.groupCiphers = {} self.pendingIncomingMessages = {} - - def receive(self, protocolTreeNode): """ :type protocolTreeNode: ProtocolTreeNode @@ -41,12 +39,23 @@ def receive(self, protocolTreeNode): if not self.processIqRegistry(protocolTreeNode): if protocolTreeNode.tag == "message": self.onMessage(protocolTreeNode) - return - elif protocolTreeNode.tag == "receipt" and protocolTreeNode["type"] == "retry": - # should bring up that message, resend it, but in upper layer? - # as it might have to be fetched from a persistent storage - pass - self.toUpper(protocolTreeNode) + elif not protocolTreeNode.tag == "receipt": + #receipts will be handled by send layer + self.toUpper(protocolTreeNode) + + # elif protocolTreeNode.tag == "iq": + # if protocolTreeNode.getChild("encr_media"): + # protocolTreeNode.addChild("media", { + # "url": protocolTreeNode["url"], + # "ip": protocolTreeNode["ip"], + # }) + # self.toUpper(protocolTreeNode) + # return + + ###### + + def onEncrMediaResult(self, resultNode): + pass def processPendingIncomingMessages(self, jid): @@ -70,8 +79,8 @@ def handleEncMessage(self, node): isGroup = node["participant"] is not None senderJid = node["participant"] if isGroup else node["from"] encNode = None - if node.getChild("enc")["v"] == "2" and senderJid not in self.v2Jids: - self.v2Jids.append(senderJid) + if node.getChild("enc")["v"] == "2" and node["from"] not in self.v2Jids: + self.v2Jids.append(node["from"]) try: if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG): self.handlePreKeyWhisperMessage(node) @@ -96,20 +105,22 @@ def handleEncMessage(self, node): self.pendingIncomingMessages[senderJid] = [] self.pendingIncomingMessages[senderJid].append(node) - self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingIncomingMessages), self.onGetKeysError) + self.getKeysFor([senderJid], lambda: self.processPendingIncomingMessages(senderJid)) + except DuplicateMessageException as e: logger.error(e) logger.warning("Going to send the delivery receipt myself !") self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"], participant=node["participant"]).toProtocolTreeNode()) except UntrustedIdentityException as e: - if(self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False)): + if self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False): logger.warning("Autotrusting identity for %s" % e.getName()) self.store.saveIdentity(e.getName(), e.getIdentityKey()) return self.handleEncMessage(node) else: logger.error(e) logger.warning("Ignoring message with untrusted identity") + def handlePreKeyWhisperMessage(self, node): pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) enc = pkMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_PKMSG) @@ -250,8 +261,6 @@ def handleContactMessage(self, originalEncNode, contactMessage): messageNode.addChild(mediaNode) self.toUpper(messageNode) - - def getSessionCipher(self, recipientId): if recipientId in self.sessionCiphers: sessionCipher = self.sessionCiphers[recipientId] diff --git a/yowsup/layers/axolotl/layer_send.py b/yowsup/layers/axolotl/layer_send.py index bc2cd2698..bd646529c 100644 --- a/yowsup/layers/axolotl/layer_send.py +++ b/yowsup/layers/axolotl/layer_send.py @@ -1,91 +1,210 @@ from yowsup.layers.protocol_messages.proto.wa_pb2 import * +from yowsup.layers.protocol_messages.proto.wa_pb2 import SenderKeyDistributionMessage as ProtoSenderKeyDistributionMessage from yowsup.layers.axolotl.protocolentities import * - -from axolotl.sessionbuilder import SessionBuilder -from axolotl.protocol.whispermessage import WhisperMessage +from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer +from yowsup.layers.protocol_groups.protocolentities import InfoGroupsIqProtocolEntity, InfoGroupsResultIqProtocolEntity from axolotl.sessioncipher import SessionCipher from axolotl.groups.groupcipher import GroupCipher from axolotl.axolotladdress import AxolotlAddress +from axolotl.protocol.whispermessage import WhisperMessage from axolotl.groups.senderkeyname import SenderKeyName +from axolotl.groups.groupsessionbuilder import GroupSessionBuilder import logging +from random import randint from .layer_base import AxolotlBaseLayer logger = logging.getLogger(__name__) class AxolotlSendLayer(AxolotlBaseLayer): + MAX_SENT_QUEUE = 100 def __init__(self): super(AxolotlSendLayer, self).__init__() self.sessionCiphers = {} + self.groupSessionBuilder = None self.groupCiphers = {} - self.pendingMessages = {} - self.skipEncJids = [] - self.v2Jids = [] + ''' + Sent messages will be put in skalti entQueue until we receive a receipt for them. + This is for handling retry receipts which requires re-encrypting and resend of the original message + As the receipt for a sent message might arrive at a different yowsup instance, + ideally the original message should be fetched from a persistent storage. + Therefore, if the original message is not in sentQueue for any reason, we will + notify the upper layers and let them handle it. + ''' + self.sentQueue = [] + + def onNewStoreSet(self, store): + if store is not None: + self.groupSessionBuilder = GroupSessionBuilder(store) def __str__(self): return "Axolotl Layer" def send(self, node): - if node.tag == "message" and node["to"] not in self.skipEncJids: - self.handlePlaintextNode(node) - return - self.toLower(node) - + if node.tag == "message" and node["to"] not in self.skipEncJids and not node.getChild("enc") and (not node.getChild("media") or node.getChild("media")["mediakey"]): + self.processPlaintextNodeAndSend(node) + # elif node.tag == "iq" and node["xmlns"] == "w:m": + # mediaNode = node.getChild("media") + # if mediaNode and mediaNode["type"] == "image": + # iqNode = IqProtocolEntity.fromProtocolTreeNode(node).toProtocolTreeNode() + # iqNode.addChild(ProtocolTreeNode( + # "encr_media", { + # "type": mediaNode["type"], + # "hash": mediaNode["hash"] + # } + # )) + # self.toLower(iqNode) + else: + self.toLower(node) - def processPendingMessages(self, jid): - if jid in self.pendingMessages: - for messageNode in self.pendingMessages[jid]: - if jid in self.skipEncJids: - self.toLower(messageNode) + def receive(self, protocolTreeNode): + if not self.processIqRegistry(protocolTreeNode): + if protocolTreeNode.tag == "receipt": + messageNode = self.getEnqueuedMessageNode(protocolTreeNode["id"]) + if not messageNode: + logger.debug("Axolotl layer does not have the message, bubbling it upwards") + self.toUpper(protocolTreeNode) + elif protocolTreeNode["type"] == "retry": + logger.info("Got retry to for message %s, and Axolotl layer has the message" % protocolTreeNode["id"]) + self.getKeysFor([protocolTreeNode["from"]], lambda: self.processPlaintextNodeAndSend(messageNode)) else: - self.handlePlaintextNode(messageNode) + #not interested in any non retry receipts, bubble upwards + self.toUpper(protocolTreeNode) - del self.pendingMessages[jid] + def processPlaintextNodeAndSend(self, node): + recipient_id = node["to"].split('@')[0] + isGroup = "-" in recipient_id + + if node.getChild("media"): + self.toLower(node) # skip media enc for now, groups and non groups + elif isGroup: + self.sendToGroup(node) + elif self.store.containsSession(recipient_id, 1): + self.sendToContact(node) + else: + self.getKeysFor([node["to"]], lambda: self.sendToContact(node), lambda: self.toLower(node)) + + + + def getPadding(self): + num = randint(1,255) + return bytearray([num] * num) + + def groupSendSequence(self): + """ + check if senderkeyrecord exists + no: - create, + - get group jids from info request + - for each jid without a session, get keys to create the session + - send message with dist key for all participants + yes: + - send skmsg without any dist key + + received retry for a participant + - request participants keys + - send message with dist key only + conversation, only for this participat + :return: + """ + + def enqueueSent(self, node): + if len(self.sentQueue) >= self.__class__.MAX_SENT_QUEUE: + logger.warn("Discarding queued node without receipt") + self.sentQueue.pop(0) + self.sentQueue.append(node) + + def getEnqueuedMessageNode(self, messageId, keepEnqueued = False): + for i in range(0, len(self.sentQueue)): + if self.sentQueue[i]["id"] == messageId: + if keepEnqueued: + return self.sentQueue[i] + return self.sentQueue.pop(i) + + + def sendEncEntities(self, node, encEntities): + mediaType = None + messageEntity = EncryptedMessageProtocolEntity(encEntities, + "text" if not mediaType else "media", + _id=node["id"], + to=node["to"], + notify=node["notify"], + timestamp=node["timestamp"], + participant=node["participant"], + offline=node["offline"], + retry=node["retry"] + ) + self.enqueueSent(node) + self.toLower(messageEntity.toProtocolTreeNode()) + + def sendToContact(self, node): + recipient_id = node["to"].split('@')[0] + cipher = self.getSessionCipher(recipient_id) + messageData = self.serializeToProtobuf(node) + self.getPadding() + + ciphertext = cipher.encrypt(messageData) + mediaType = node.getChild("media")["type"] if node.getChild("media") else None + + return self.sendEncEntities(node, [EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG, 2, ciphertext.serialize(), mediaType)]) + + def sendToGroupWithSessions(self, node, jidsNeedSenderKey = None): + jidsNeedSenderKey = jidsNeedSenderKey or [] + groupJid = node["to"] + ownNumber = self.getLayerInterface(YowAuthenticationProtocolLayer).getUsername(False) + senderKeyName = SenderKeyName(groupJid, AxolotlAddress(ownNumber, 0)) + cipher = self.getGroupCipher(groupJid, ownNumber) + encEntities = [] + senderKeyDistributionMessage = self.groupSessionBuilder.create(senderKeyName) + for jid in jidsNeedSenderKey: + sessionCipher = self.getSessionCipher(jid.split('@')[0]) + serialized = self.serializeSenderKeyDistributionMessageToProtobuf(node["to"], senderKeyDistributionMessage) + ciphertext = sessionCipher.encrypt(serialized + self.getPadding()) + encEntities.append( + EncProtocolEntity( + EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG + , 2, ciphertext.serialize(), jid=jid + ) + ) + + messageData = self.serializeToProtobuf(node) + ciphertext = cipher.encrypt(messageData + self.getPadding()) + mediaType = node.getChild("media")["type"] if node.getChild("media") else None + + encEntities.append(EncProtocolEntity(EncProtocolEntity.TYPE_SKMSG, 2, ciphertext, mediaType)) + + self.sendEncEntities(node, encEntities) + + def ensureSessionsAndSendToGroup(self, node, jids): + jidsNoSession = [] + for jid in jids: + if not self.store.containsSession(jid.split('@')[0], 1): + jidsNoSession.append(jid) + + if len(jidsNoSession): + self.getKeysFor(jidsNoSession, lambda: self.sendToGroupWithSessions(node, jids)) + else: + self.sendToGroupWithSessions(node, jids) + def sendToGroup(self, node): + groupJid = node["to"] + ownNumber = self.getLayerInterface(YowAuthenticationProtocolLayer).getUsername(False) + ownJid = self.getLayerInterface(YowAuthenticationProtocolLayer).getUsername(True) + senderKeyName = SenderKeyName(node["to"], AxolotlAddress(ownNumber, 0)) + senderKeyRecord = self.store.loadSenderKey(senderKeyName) - #### handling message types + def sendToGroup(resultNode, requestEntity): + groupInfo = InfoGroupsResultIqProtocolEntity.fromProtocolTreeNode(resultNode) + jids = groupInfo.getParticipants().keys() + return self.ensureSessionsAndSendToGroup(node, [jid for jid in jids if jid != ownJid]) - def handlePlaintextNode(self, node): - recipient_id = node["to"].split('@')[0] - v2 = node["to"] in self.v2Jids - if not self.store.containsSession(recipient_id, 1): - entity = GetKeysIqProtocolEntity([node["to"]]) - if node["to"] not in self.pendingMessages: - self.pendingMessages[node["to"]] = [] - self.pendingMessages[node["to"]].append(node) - - self._sendIq(entity, lambda a, b: self.onGetKeysResult(a, b, self.processPendingMessages), self.onGetKeysError) - elif v2 or not node.getChild("media"): #media enc is only for v2 messsages - messageData = self.serializeToProtobuf(node) if v2 else node.getChild("body").getData() - if messageData: - sessionCipher = self.getSessionCipher(recipient_id) - ciphertext = sessionCipher.encrypt(messageData) - - mediaType = node.getChild("media")["type"] if node.getChild("media") else None - - encEntity = EncryptedMessageProtocolEntity( - [ - EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG , - 2 if v2 else 1, - ciphertext.serialize(), mediaType)], - "text" if not mediaType else "media", - _id= node["id"], - to = node["to"], - notify = node["notify"], - timestamp= node["timestamp"], - participant=node["participant"], - offline=node["offline"], - retry=node["retry"] - ) - self.toLower(encEntity.toProtocolTreeNode()) - else: #case of unserializable messages (audio, video) ? - self.toLower(node) + if senderKeyRecord.isEmpty(): + self.groupSessionBuilder.create(senderKeyName) + groupInfoIq = InfoGroupsIqProtocolEntity(groupJid) + self._sendIq(groupInfoIq, sendToGroup) else: - self.toLower(node) + self.sendToGroupWithSessions(node) def serializeToProtobuf(self, node): if node.getChild("body"): @@ -153,28 +272,14 @@ def serializeUrlToProtobuf(self, node): def serializeDocumentToProtobuf(self, node): pass - def onGetKeysResult(self, resultNode, getKeysEntity, processPendingFn): - entity = ResultGetKeysIqProtocolEntity.fromProtocolTreeNode(resultNode) - - resultJids = entity.getJids() - for jid in getKeysEntity.getJids(): - - if jid not in resultJids: - self.skipEncJids.append(jid) - self.processPendingMessages(jid) - continue - - recipient_id = jid.split('@')[0] - preKeyBundle = entity.getPreKeyBundleFor(jid) - - sessionBuilder = SessionBuilder(self.store, self.store, self.store, - self.store, recipient_id, 1) - sessionBuilder.processPreKeyBundle(preKeyBundle) - - processPendingFn(jid) - - def onGetKeysError(self, errorNode, getKeysEntity): - pass + def serializeSenderKeyDistributionMessageToProtobuf(self, groupId, senderKeyDistributionMessage): + m = Message() + sender_key_distribution_message = ProtoSenderKeyDistributionMessage() + sender_key_distribution_message.groupId = groupId + sender_key_distribution_message.axolotl_sender_key_distribution_message = senderKeyDistributionMessage.serialize() + m.sender_key_distribution_message.MergeFrom(sender_key_distribution_message) + # m.conversation = text + return m.SerializeToString() ### def getSessionCipher(self, recipientId): @@ -187,7 +292,7 @@ def getSessionCipher(self, recipientId): return sessionCipher def getGroupCipher(self, groupId, senderId): - senderKeyName = SenderKeyName(groupId, AxolotlAddress(senderId, 1)) + senderKeyName = SenderKeyName(groupId, AxolotlAddress(senderId, 0)) if senderKeyName in self.groupCiphers: groupCipher = self.groupCiphers[senderKeyName] else: diff --git a/yowsup/layers/axolotl/protocolentities/enc.py b/yowsup/layers/axolotl/protocolentities/enc.py index 6d5655809..e267a41dd 100644 --- a/yowsup/layers/axolotl/protocolentities/enc.py +++ b/yowsup/layers/axolotl/protocolentities/enc.py @@ -5,13 +5,15 @@ class EncProtocolEntity(ProtocolEntity): TYPE_MSG = "msg" TYPE_SKMSG = "skmsg" TYPES = (TYPE_PKMSG, TYPE_MSG, TYPE_SKMSG) - def __init__(self, type, version, data, mediaType = None): + + def __init__(self, type, version, data, mediaType = None, jid = None): assert type in self.__class__.TYPES, "Unknown message enc type %s" % type super(EncProtocolEntity, self).__init__("enc") self.type = type self.version = int(version) self.data = data self.mediaType = mediaType + self.jid = jid def getType(self): return self.type @@ -25,11 +27,17 @@ def getData(self): def getMediaType(self): return self.mediaType + def getJid(self): + return self.jid + def toProtocolTreeNode(self): attribs = {"type": self.type, "v": str(self.version)} if self.mediaType: attribs["mediatype"] = self.mediaType - return ProtocolTreeNode("enc", attribs, data = self.data) + encNode = ProtocolTreeNode("enc", attribs, data = self.data) + if self.jid: + return ProtocolTreeNode("to", {"jid": self.jid}, [encNode]) + return encNode @staticmethod def fromProtocolTreeNode(node): diff --git a/yowsup/layers/axolotl/protocolentities/message_encrypted.py b/yowsup/layers/axolotl/protocolentities/message_encrypted.py index cfe9d2c7b..1e23154ea 100644 --- a/yowsup/layers/axolotl/protocolentities/message_encrypted.py +++ b/yowsup/layers/axolotl/protocolentities/message_encrypted.py @@ -31,8 +31,16 @@ def getEnc(self, encType): def toProtocolTreeNode(self): node = super(EncryptedMessageProtocolEntity, self).toProtocolTreeNode() + participantsNode = ProtocolTreeNode("participants") for key, enc in self.encEntities.items(): - node.addChild(enc.toProtocolTreeNode()) + encNode = enc.toProtocolTreeNode() + if encNode.tag == "to": + participantsNode.addChild(encNode) + else: + node.addChild(encNode) + + if len(participantsNode.getAllChildren()): + node.addChild(participantsNode) return node From d267071813dcc02f1511649c9d519ffab041e89f Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sat, 16 Apr 2016 21:39:32 +0200 Subject: [PATCH 44/68] Fixes to group send --- yowsup/layers/axolotl/layer_send.py | 24 +++++++++---------- .../protocolentities/message_encrypted.py | 13 ++++------ 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/yowsup/layers/axolotl/layer_send.py b/yowsup/layers/axolotl/layer_send.py index bd646529c..683214366 100644 --- a/yowsup/layers/axolotl/layer_send.py +++ b/yowsup/layers/axolotl/layer_send.py @@ -69,7 +69,7 @@ def receive(self, protocolTreeNode): self.toUpper(protocolTreeNode) elif protocolTreeNode["type"] == "retry": logger.info("Got retry to for message %s, and Axolotl layer has the message" % protocolTreeNode["id"]) - self.getKeysFor([protocolTreeNode["from"]], lambda: self.processPlaintextNodeAndSend(messageNode)) + self.getKeysFor([protocolTreeNode["participant"] or protocolTreeNode["from"]], lambda: self.processPlaintextNodeAndSend(messageNode)) else: #not interested in any non retry receipts, bubble upwards self.toUpper(protocolTreeNode) @@ -155,17 +155,18 @@ def sendToGroupWithSessions(self, node, jidsNeedSenderKey = None): senderKeyName = SenderKeyName(groupJid, AxolotlAddress(ownNumber, 0)) cipher = self.getGroupCipher(groupJid, ownNumber) encEntities = [] - senderKeyDistributionMessage = self.groupSessionBuilder.create(senderKeyName) - for jid in jidsNeedSenderKey: - sessionCipher = self.getSessionCipher(jid.split('@')[0]) - serialized = self.serializeSenderKeyDistributionMessageToProtobuf(node["to"], senderKeyDistributionMessage) - ciphertext = sessionCipher.encrypt(serialized + self.getPadding()) - encEntities.append( - EncProtocolEntity( - EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG - , 2, ciphertext.serialize(), jid=jid + if len(jidsNeedSenderKey): + senderKeyDistributionMessage = self.groupSessionBuilder.create(senderKeyName) + for jid in jidsNeedSenderKey: + sessionCipher = self.getSessionCipher(jid.split('@')[0]) + serialized = self.serializeSenderKeyDistributionMessageToProtobuf(node["to"], senderKeyDistributionMessage) + ciphertext = sessionCipher.encrypt(serialized + self.getPadding()) + encEntities.append( + EncProtocolEntity( + EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG + , 2, ciphertext.serialize(), jid=jid + ) ) - ) messageData = self.serializeToProtobuf(node) ciphertext = cipher.encrypt(messageData + self.getPadding()) @@ -200,7 +201,6 @@ def sendToGroup(resultNode, requestEntity): return self.ensureSessionsAndSendToGroup(node, [jid for jid in jids if jid != ownJid]) if senderKeyRecord.isEmpty(): - self.groupSessionBuilder.create(senderKeyName) groupInfoIq = InfoGroupsIqProtocolEntity(groupJid) self._sendIq(groupInfoIq, sendToGroup) else: diff --git a/yowsup/layers/axolotl/protocolentities/message_encrypted.py b/yowsup/layers/axolotl/protocolentities/message_encrypted.py index 1e23154ea..86cacf4ee 100644 --- a/yowsup/layers/axolotl/protocolentities/message_encrypted.py +++ b/yowsup/layers/axolotl/protocolentities/message_encrypted.py @@ -18,21 +18,18 @@ def __init__(self, encEntities, _type, _id = None, _from = None, to = None, not self.setEncEntities(encEntities) def setEncEntities(self, encEntities): - self.encEntities = {} assert len(encEntities), "Must have at least 1 enc entity" - for enc in encEntities: - self.encEntities[enc.type] = enc + self.encEntities = encEntities def getEnc(self, encType): - if encType in self.encEntities: - return self.encEntities[encType] - - return None + for enc in self.encEntities: + if enc.type == encType: + return enc def toProtocolTreeNode(self): node = super(EncryptedMessageProtocolEntity, self).toProtocolTreeNode() participantsNode = ProtocolTreeNode("participants") - for key, enc in self.encEntities.items(): + for enc in self.encEntities: encNode = enc.toProtocolTreeNode() if encNode.tag == "to": participantsNode.addChild(encNode) From 76f3df95f4bdc4bfa7534360cb9c651504b74a8c Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 17 Apr 2016 13:22:38 +0200 Subject: [PATCH 45/68] Handle Sender Key message carrying data from a retry --- yowsup/layers/axolotl/layer_receive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index cf3fc44f9..c1d03a207 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -181,7 +181,8 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): if m.HasField("sender_key_distribution_message"): axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) - elif m.HasField("conversation"): + + if m.HasField("conversation"): self.handleConversationMessage(node, m.conversation) elif m.HasField("contact_message"): self.handleContactMessage(node, m.contact_message) From 3accd149d7a8c656e779fa9b42ec68e551ae4cd2 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 17 Apr 2016 13:31:03 +0200 Subject: [PATCH 46/68] Fixed group receive enc="msg" only --- yowsup/layers/axolotl/layer_receive.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index c1d03a207..2684e1d76 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -167,6 +167,7 @@ def handleSenderKeyMessage(self, node): def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): node = encMessageProtocolEntity.toProtocolTreeNode() m = Message() + handled = False try: m.ParseFromString(serializedData) except: @@ -179,20 +180,27 @@ def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): raise ValueError("Empty message") if m.HasField("sender_key_distribution_message"): + handled = True axolotlAddress = AxolotlAddress(encMessageProtocolEntity.getParticipant(False), 0) self.handleSenderKeyDistributionMessage(m.sender_key_distribution_message, axolotlAddress) if m.HasField("conversation"): + handled = True self.handleConversationMessage(node, m.conversation) elif m.HasField("contact_message"): + handled = True self.handleContactMessage(node, m.contact_message) elif m.HasField("url_message"): + handled = True self.handleUrlMessage(node, m.url_message) elif m.HasField("location_message"): + handled = True self.handleLocationMessage(node, m.location_message) elif m.HasField("image_message"): + handled = True self.handleImageMessage(node, m.image_message) - else: + + if not handled: print(m) raise ValueError("Unhandled") From 41e75a4e2648e04031ceecdca4c42a9caa6ae3b7 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Mon, 18 Apr 2016 02:15:21 +0200 Subject: [PATCH 47/68] Cleanups post single AxolotlLayer remove --- yowsup/demos/cli/layer.py | 9 ------ yowsup/demos/contacts/stack.py | 44 +++++++--------------------- yowsup/layers/axolotl/__init__.py | 1 - yowsup/layers/interface/interface.py | 5 ---- 4 files changed, 11 insertions(+), 48 deletions(-) diff --git a/yowsup/demos/cli/layer.py b/yowsup/demos/cli/layer.py index 04a094917..d00b51918 100644 --- a/yowsup/demos/cli/layer.py +++ b/yowsup/demos/cli/layer.py @@ -8,11 +8,9 @@ import datetime import os import logging -from yowsup.layers.protocol_receipts.protocolentities import * from yowsup.layers.protocol_groups.protocolentities import * from yowsup.layers.protocol_presence.protocolentities import * from yowsup.layers.protocol_messages.protocolentities import * -from yowsup.layers.protocol_acks.protocolentities import * from yowsup.layers.protocol_ib.protocolentities import * from yowsup.layers.protocol_iq.protocolentities import * from yowsup.layers.protocol_contacts.protocolentities import * @@ -343,13 +341,6 @@ def keys_get(self, jids): entity = GetKeysIqProtocolEntity(jids) self.toLower(entity) - @clicmd("Send prekeys") - def keys_set(self): - with AxolotlOptionalModule(failMessage = self.__class__.FAIL_OPT_AXOLOTL) as axoOptMod: - from yowsup.layers.axolotl import YowAxolotlLayer - if self.assertConnected(): - self.broadcastEvent(YowLayerEvent(YowAxolotlLayer.EVENT_PREKEYS_SET)) - @clicmd("Send init seq") def seq(self): priv = PrivacyListIqProtocolEntity() diff --git a/yowsup/demos/contacts/stack.py b/yowsup/demos/contacts/stack.py index 0ecb29454..26228ec35 100644 --- a/yowsup/demos/contacts/stack.py +++ b/yowsup/demos/contacts/stack.py @@ -1,15 +1,10 @@ -from yowsup.stacks import YowStack from .layer import SyncLayer + +from yowsup.stacks import YowStackBuilder +from yowsup.layers.auth import AuthError from yowsup.layers import YowLayerEvent -from yowsup.layers.auth import YowCryptLayer, YowAuthenticationProtocolLayer, AuthError -from yowsup.layers.coder import YowCoderLayer -from yowsup.layers.network import YowNetworkLayer -from yowsup.layers.stanzaregulator import YowStanzaRegulator -from yowsup.layers.protocol_receipts import YowReceiptProtocolLayer -from yowsup.layers.protocol_acks import YowAckProtocolLayer -from yowsup.layers.logger import YowLoggerLayer -from yowsup.layers.protocol_contacts import YowContactsIqProtocolLayer -from yowsup.layers import YowParallelLayer +from yowsup.layers.auth import YowAuthenticationProtocolLayer +from yowsup.layers.network import YowNetworkLayer class YowsupSyncStack(object): def __init__(self, credentials, contacts, encryptionEnabled = False): @@ -19,30 +14,13 @@ def __init__(self, credentials, contacts, encryptionEnabled = False): :param encryptionEnabled: :return: """ - if encryptionEnabled: - from yowsup.layers.axolotl import YowAxolotlLayer - layers = ( - SyncLayer, - YowParallelLayer([YowAuthenticationProtocolLayer, YowContactsIqProtocolLayer, YowReceiptProtocolLayer, YowAckProtocolLayer]), - YowAxolotlLayer, - YowLoggerLayer, - YowCoderLayer, - YowCryptLayer, - YowStanzaRegulator, - YowNetworkLayer - ) - else: - layers = ( - SyncLayer, - YowParallelLayer([YowAuthenticationProtocolLayer, YowContactsIqProtocolLayer, YowReceiptProtocolLayer, YowAckProtocolLayer]), - YowLoggerLayer, - YowCoderLayer, - YowCryptLayer, - YowStanzaRegulator, - YowNetworkLayer - ) + stackBuilder = YowStackBuilder() + + self.stack = stackBuilder \ + .pushDefaultLayers(encryptionEnabled) \ + .push(SyncLayer) \ + .build() - self.stack = YowStack(layers) self.stack.setProp(SyncLayer.PROP_CONTACTS, contacts) self.stack.setProp(YowAuthenticationProtocolLayer.PROP_PASSIVE, True) self.stack.setCredentials(credentials) diff --git a/yowsup/layers/axolotl/__init__.py b/yowsup/layers/axolotl/__init__.py index 3b53b7894..91f204cef 100644 --- a/yowsup/layers/axolotl/__init__.py +++ b/yowsup/layers/axolotl/__init__.py @@ -1,4 +1,3 @@ -# from .layer import YowAxolotlLayer from .layer_send import AxolotlSendLayer from .layer_control import AxolotlControlLayer from .layer_receive import AxolotlReceivelayer diff --git a/yowsup/layers/interface/interface.py b/yowsup/layers/interface/interface.py index 763309e11..681e7d5e6 100644 --- a/yowsup/layers/interface/interface.py +++ b/yowsup/layers/interface/interface.py @@ -2,13 +2,8 @@ from yowsup.layers.protocol_iq.protocolentities import IqProtocolEntity from yowsup.layers.network import YowNetworkLayer from yowsup.layers.auth import YowAuthenticationProtocolLayer -from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity -from yowsup.layers.protocol_acks.protocolentities import IncomingAckProtocolEntity -from yowsup.layers.axolotl.layer import YowAxolotlLayer from yowsup.layers.protocol_media.protocolentities.iq_requestupload import RequestUploadIqProtocolEntity -from yowsup.layers.protocol_media.protocolentities.iq_requestupload_result import ResultRequestUploadIqProtocolEntity from yowsup.layers.protocol_media.mediauploader import MediaUploader -from yowsup.layers.axolotl.layer import YowAxolotlLayer import inspect class ProtocolEntityCallback(object): From 739d734006154f1f8530f0bea2679896fe0352ea Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Mon, 18 Apr 2016 02:27:51 +0200 Subject: [PATCH 48/68] Added missing import --- yowsup/layers/axolotl/layer_control.py | 1 + yowsup/layers/axolotl/props.py | 1 + .../receipt_incoming_retry.py | 57 +++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 yowsup/layers/axolotl/props.py create mode 100644 yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py diff --git a/yowsup/layers/axolotl/layer_control.py b/yowsup/layers/axolotl/layer_control.py index 04cec6b7f..027bff5b9 100644 --- a/yowsup/layers/axolotl/layer_control.py +++ b/yowsup/layers/axolotl/layer_control.py @@ -4,6 +4,7 @@ from axolotl.util.keyhelper import KeyHelper from yowsup.layers.axolotl.protocolentities import * from yowsup.layers.auth.layer_authentication import YowAuthenticationProtocolLayer +from yowsup.layers.protocol_acks.protocolentities import OutgoingAckProtocolEntity from axolotl.util.hexutil import HexUtil from axolotl.ecc.curve import Curve import logging diff --git a/yowsup/layers/axolotl/props.py b/yowsup/layers/axolotl/props.py new file mode 100644 index 000000000..685631c45 --- /dev/null +++ b/yowsup/layers/axolotl/props.py @@ -0,0 +1 @@ +PROP_IDENTITY_AUTOTRUST = "org.openwhatsapp.yowsup.prop.axolotl.INDENTITY_AUTOTRUST" \ No newline at end of file diff --git a/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py b/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py new file mode 100644 index 000000000..ede7adfeb --- /dev/null +++ b/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py @@ -0,0 +1,57 @@ +from yowsup.structs import ProtocolTreeNode +from yowsup.layers.protocol_receipts.protocolentities import IncomingReceiptProtocolEntity +from yowsup.layers.axolotl.protocolentities.iq_keys_get_result import ResultGetKeysIqProtocolEntity +class RetryIncomingReceiptProtocolEntity(IncomingReceiptProtocolEntity): + + ''' + + + + + HEX:xxxxxxxxx + + + ''' + + def __init__(self, _id, jid, remoteRegistrationId, receiptTimestamp, retryTimestamp, v = 1, count = 1, participant = None, offline = None): + super(RetryIncomingReceiptProtocolEntity, self).__init__(_id, jid, receiptTimestamp, offline=offline, type="retry", participant=participant) + self.setRetryData(remoteRegistrationId, v,count, retryTimestamp) + + def setRetryData(self, remoteRegistrationId, v, count, retryTimestamp): + self.remoteRegistrationId = remoteRegistrationId + self.v = v + self.count = count + self.retryTimestamp = retryTimestamp + + def toProtocolTreeNode(self): + node = super(RetryIncomingReceiptProtocolEntity, self).toProtocolTreeNode() + + retry = ProtocolTreeNode("retry", { + "count": str(self.count), + "id": self.getId(), + "v": str(self.v), + "t": str(self.retryTimestamp) + }) + node.addChild(retry) + registration = ProtocolTreeNode("registration", data=ResultGetKeysIqProtocolEntity._intToBytes(self.remoteRegistrationId)) + node.addChild(registration) + return node + + def getRetryCount(self): + return self.count + + def getRetryJid(self): + return self.getParticipant() or self.getFrom() + + def __str__(self): + out = super(RetryIncomingReceiptProtocolEntity, self).__str__() + return out + + @staticmethod + def fromProtocolTreeNode(node): + entity = IncomingReceiptProtocolEntity.fromProtocolTreeNode(node) + entity.__class__ = RetryIncomingReceiptProtocolEntity + retryNode = node.getChild("retry") + entity.setRetryData(ResultGetKeysIqProtocolEntity._bytesToInt(node.getChild("registration").data), retryNode["v"], retryNode["count"], retryNode["t"]) + + return entity From 218d2a7580a8a5c29b07f8f8de109f304b987d50 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 12:07:08 +0200 Subject: [PATCH 49/68] Autorecover connection initial commit --- yowsup/layers/auth/layer_authentication.py | 11 +++++---- yowsup/layers/interface/interface.py | 28 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/yowsup/layers/auth/layer_authentication.py b/yowsup/layers/auth/layer_authentication.py index 1183391f5..90d6c693b 100644 --- a/yowsup/layers/auth/layer_authentication.py +++ b/yowsup/layers/auth/layer_authentication.py @@ -9,6 +9,7 @@ from yowsup.common.tools import StorageTools from yowsup.env import YowsupEnv from .layer_interface_authentication import YowAuthenticationProtocolLayerInterface +from .protocolentities import StreamErrorProtocolEntity import base64 class YowAuthenticationProtocolLayer(YowProtocolLayer): @@ -84,12 +85,12 @@ def handleChallenge(self, node): self._sendResponse(nodeEntity.getNonce()) def handleStreamError(self, node): - if node.getChild("text"): - nodeEntity = StreamErrorConflictProtocolEntity.fromProtocolTreeNode(node) - elif node.getChild("ack"): - nodeEntity = StreamErrorAckProtocolEntity.fromProtocolTreeNode(node) - else: + nodeEntity = StreamErrorProtocolEntity.fromProtocolTreeNode(node) + errorType = nodeEntity.getErrorType() + + if not errorType: raise AuthError("Unhandled stream:error node:\n%s" % node) + self.toUpper(nodeEntity) ##senders diff --git a/yowsup/layers/interface/interface.py b/yowsup/layers/interface/interface.py index 681e7d5e6..8f3ef0bc2 100644 --- a/yowsup/layers/interface/interface.py +++ b/yowsup/layers/interface/interface.py @@ -4,7 +4,11 @@ from yowsup.layers.auth import YowAuthenticationProtocolLayer from yowsup.layers.protocol_media.protocolentities.iq_requestupload import RequestUploadIqProtocolEntity from yowsup.layers.protocol_media.mediauploader import MediaUploader +from yowsup.layers.network.layer import YowNetworkLayer +from yowsup.layers import EventCallback import inspect +import logging +logger = logging.getLogger(__name__) class ProtocolEntityCallback(object): def __init__(self, entityType): @@ -17,8 +21,11 @@ def __call__(self, fn): class YowInterfaceLayer(YowLayer): + PROP_RECONNECT_ON_STREAM_ERR = "org.openwhatsapp.yowsup.prop.interface.reconnect_on_stream_error" + def __init__(self): super(YowInterfaceLayer, self).__init__() + self.reconnect = False self.entity_callbacks = {} self.iqRegistry = {} # self.receiptsRegistry = {} @@ -74,6 +81,27 @@ def receive(self, entity): else: self.toUpper(entity) + @ProtocolEntityCallback("stream:error") + def onStreamError(self, streamErrorEntity): + logger.error(streamErrorEntity) + if self.getProp(self.__class__.PROP_RECONNECT_ON_STREAM_ERR, True): + logger.info("Initiating reconnect") + self.reconnect = True + self.disconnect() + else: + logger.warn("No reconnecting because property %s is not set" % self.__class__.PROP_RECONNECT_ON_STREAM_ERR) + self.toUpper(streamErrorEntity) + + @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) + def onConnected(self, yowLayerEvent): + self.reconnect = False + + @EventCallback(YowNetworkLayer.EVENT_STATE_DISCONNECTED) + def onDisconnected(self, yowLayerEvent): + if self.reconnect: + self.reconnect = False + self.connect() + def _sendMediaMessage(self, builder, success, error = None, progress = None): # axolotlIface = self.getLayerInterface(YowAxolotlLayer) # if axolotlIface: From 822f45b090979b77067ab274e6d80ad24a4da4ac Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 17:45:59 +0200 Subject: [PATCH 50/68] Fully working group encryption support Can now recover failed enc/dec scenarios (no session, no senderkey, untrustedidentitiy, bad mac, ...etc) Sent messages are cached in axolotl layer, in case we receive a retry node for them Respond to retry nodes, re-request associated keys and rencrypt --- yowsup/demos/cli/layer.py | 2 +- yowsup/demos/cli/stack.py | 5 +- yowsup/layers/axolotl/layer_base.py | 25 ++++- yowsup/layers/axolotl/layer_receive.py | 39 ++++--- yowsup/layers/axolotl/layer_send.py | 106 +++++++++++------- .../axolotl/protocolentities/__init__.py | 1 + .../receipt_outgoing_retry.py | 54 ++++----- .../protocolentities/ack_outgoing.py | 1 - .../protocolentities/receipt_incoming.py | 5 +- 9 files changed, 142 insertions(+), 96 deletions(-) diff --git a/yowsup/demos/cli/layer.py b/yowsup/demos/cli/layer.py index d00b51918..e6b319d22 100644 --- a/yowsup/demos/cli/layer.py +++ b/yowsup/demos/cli/layer.py @@ -23,7 +23,6 @@ from yowsup.common.optionalmodules import PILOptionalModule, AxolotlOptionalModule logger = logging.getLogger(__name__) - class YowsupCliLayer(Cli, YowInterfaceLayer): PROP_RECEIPT_AUTO = "org.openwhatsapp.yowsup.prop.cli.autoreceipt" PROP_RECEIPT_KEEPALIVE = "org.openwhatsapp.yowsup.prop.cli.keepalive" @@ -76,6 +75,7 @@ def jidToAlias(self, jid): def setCredentials(self, username, password): self.getLayerInterface(YowAuthenticationProtocolLayer).setCredentials(username, password) + return "%s@s.whatsapp.net" % username @EventCallback(EVENT_START) def onStart(self, layerEvent): diff --git a/yowsup/demos/cli/stack.py b/yowsup/demos/cli/stack.py index fa5c84fdc..4b37dfea4 100644 --- a/yowsup/demos/cli/stack.py +++ b/yowsup/demos/cli/stack.py @@ -2,8 +2,7 @@ from .layer import YowsupCliLayer from yowsup.layers.auth import AuthError from yowsup.layers import YowLayerEvent -from yowsup.layers.auth import YowAuthenticationProtocolLayer -from yowsup.layers.axolotl import AxolotlReceivelayer +from yowsup.layers.axolotl.props import PROP_IDENTITY_AUTOTRUST import sys class YowsupCliStack(object): @@ -17,7 +16,7 @@ def __init__(self, credentials, encryptionEnabled = True): # self.stack.setCredentials(credentials) self.stack.setCredentials(credentials) - self.stack.setProp(AxolotlReceivelayer.PROP_IDENTITY_AUTOTRUST, True) + self.stack.setProp(PROP_IDENTITY_AUTOTRUST, True) def start(self): print("Yowsup Cli client\n==================\nType /help for available commands\n") diff --git a/yowsup/layers/axolotl/layer_base.py b/yowsup/layers/axolotl/layer_base.py index aa9629253..acbea2359 100644 --- a/yowsup/layers/axolotl/layer_base.py +++ b/yowsup/layers/axolotl/layer_base.py @@ -5,7 +5,11 @@ from yowsup.layers.axolotl.protocolentities import * from axolotl.sessionbuilder import SessionBuilder +from axolotl.untrustedidentityexception import UntrustedIdentityException +from yowsup.layers.axolotl.props import PROP_IDENTITY_AUTOTRUST +import logging +logger = logging.getLogger(__name__) class AxolotlBaseLayer(YowProtocolLayer): _DB = "axolotl.db" def __init__(self): @@ -39,10 +43,12 @@ def store(self, store): self._store = store self.onNewStoreSet(self._store) - def getKeysFor(self, jids, successClbk, errorClbk=None): + def getKeysFor(self, jids, resultClbk, errorClbk = None): def onSuccess(resultNode, getKeysEntity): entity = ResultGetKeysIqProtocolEntity.fromProtocolTreeNode(resultNode) resultJids = entity.getJids() + successJids = [] + errorJids = {} #jid -> exception for jid in getKeysEntity.getJids(): if jid not in resultJids: @@ -52,13 +58,24 @@ def onSuccess(resultNode, getKeysEntity): recipient_id = jid.split('@')[0] preKeyBundle = entity.getPreKeyBundleFor(jid) sessionBuilder = SessionBuilder(self.store, self.store, self.store, self.store, recipient_id, 1) - sessionBuilder.processPreKeyBundle(preKeyBundle) + try: + sessionBuilder.processPreKeyBundle(preKeyBundle) + successJids.append(jid) + except UntrustedIdentityException as e: + if self.getProp(PROP_IDENTITY_AUTOTRUST, False): + logger.warning("Autotrusting identity for %s" % e.getName()) + self.store.saveIdentity(e.getName(), e.getIdentityKey()) + successJids.append(jid) + else: + errorJids[jid] = e + logger.error(e) + logger.warning("Ignoring message with untrusted identity") - successClbk() + resultClbk(successJids, errorJids) def onError(errorNode, getKeysEntity): if errorClbk: - errorClbk() + errorClbk(errorNode, getKeysEntity) entity = GetKeysIqProtocolEntity(jids) self._sendIq(entity, onSuccess, onError=onError) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index 2684e1d76..3e2c5ad0a 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -4,6 +4,7 @@ from yowsup.layers.protocol_messages.proto.wa_pb2 import * from yowsup.layers.axolotl.protocolentities import * from yowsup.structs import ProtocolTreeNode +from yowsup.layers.axolotl.props import PROP_IDENTITY_AUTOTRUST from axolotl.protocol.prekeywhispermessage import PreKeyWhisperMessage from axolotl.protocol.whispermessage import WhisperMessage @@ -24,13 +25,12 @@ logger = logging.getLogger(__name__) class AxolotlReceivelayer(AxolotlBaseLayer): - PROP_IDENTITY_AUTOTRUST = "org.openwhatsapp.yowsup.prop.axolotl.INDENTITY_AUTOTRUST" def __init__(self): super(AxolotlReceivelayer, self).__init__() self.v2Jids = [] #people we're going to send v2 enc messages self.sessionCiphers = {} self.groupCiphers = {} - self.pendingIncomingMessages = {} + self.pendingIncomingMessages = {} #(jid, participantJid?) => message def receive(self, protocolTreeNode): """ @@ -58,12 +58,13 @@ def onEncrMediaResult(self, resultNode): pass - def processPendingIncomingMessages(self, jid): - if jid in self.pendingIncomingMessages: - for messageNode in self.pendingIncomingMessages[jid]: + def processPendingIncomingMessages(self, jid, participantJid = None): + conversationIdentifier = (jid, participantJid) + if conversationIdentifier in self.pendingIncomingMessages: + for messageNode in self.pendingIncomingMessages[conversationIdentifier]: self.onMessage(messageNode) - del self.pendingIncomingMessages[jid] + del self.pendingIncomingMessages[conversationIdentifier] ##### handling received data ##### @@ -90,22 +91,25 @@ def handleEncMessage(self, node): self.handleSenderKeyMessage(node) except InvalidMessageException as e: logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) - retry.setRegData(self.store.getLocalRegistrationId()) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node, self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) except InvalidKeyIdException as e: logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) - retry.setRegData(self.store.getLocalRegistrationId()) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node,self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) except NoSessionException as e: logger.error(e) entity = GetKeysIqProtocolEntity([senderJid]) - if senderJid not in self.pendingIncomingMessages: - self.pendingIncomingMessages[senderJid] = [] - self.pendingIncomingMessages[senderJid].append(node) - self.getKeysFor([senderJid], lambda: self.processPendingIncomingMessages(senderJid)) + conversationIdentifier = (node["from"], node["participant"]) + + if conversationIdentifier not in self.pendingIncomingMessages: + self.pendingIncomingMessages[conversationIdentifier] = [] + self.pendingIncomingMessages[conversationIdentifier].append(node) + + successFn = lambda successJids, b: self.processPendingIncomingMessages(*conversationIdentifier) if len(successJids) else None + + self.getKeysFor([senderJid], successFn) except DuplicateMessageException as e: logger.error(e) @@ -113,7 +117,7 @@ def handleEncMessage(self, node): self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"], participant=node["participant"]).toProtocolTreeNode()) except UntrustedIdentityException as e: - if self.getProp(self.__class__.PROP_IDENTITY_AUTOTRUST, False): + if self.getProp(PROP_IDENTITY_AUTOTRUST, False): logger.warning("Autotrusting identity for %s" % e.getName()) self.store.saveIdentity(e.getName(), e.getIdentityKey()) return self.handleEncMessage(node) @@ -157,13 +161,12 @@ def handleSenderKeyMessage(self, node): plaintext = groupCipher.decrypt(enc.getData()) padding = ord(plaintext[-1]) & 0xFF self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) + except NoSessionException as e: logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node) - retry.setRegData(self.store.getLocalRegistrationId()) + retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node, self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) - def parseAndHandleMessageProto(self, encMessageProtocolEntity, serializedData): node = encMessageProtocolEntity.toProtocolTreeNode() m = Message() diff --git a/yowsup/layers/axolotl/layer_send.py b/yowsup/layers/axolotl/layer_send.py index 683214366..f5937bd8b 100644 --- a/yowsup/layers/axolotl/layer_send.py +++ b/yowsup/layers/axolotl/layer_send.py @@ -63,29 +63,38 @@ def send(self, node): def receive(self, protocolTreeNode): if not self.processIqRegistry(protocolTreeNode): if protocolTreeNode.tag == "receipt": - messageNode = self.getEnqueuedMessageNode(protocolTreeNode["id"]) + ''' + Going to keep all group message enqueued, as we get receipts from each participant + So can't just remove it on first receipt. Therefore, the MAX queue length mechanism should better be working + ''' + messageNode = self.getEnqueuedMessageNode(protocolTreeNode["id"], protocolTreeNode["participant"] is not None) if not messageNode: logger.debug("Axolotl layer does not have the message, bubbling it upwards") self.toUpper(protocolTreeNode) elif protocolTreeNode["type"] == "retry": logger.info("Got retry to for message %s, and Axolotl layer has the message" % protocolTreeNode["id"]) - self.getKeysFor([protocolTreeNode["participant"] or protocolTreeNode["from"]], lambda: self.processPlaintextNodeAndSend(messageNode)) + retryReceiptEntity = RetryIncomingReceiptProtocolEntity.fromProtocolTreeNode(protocolTreeNode) + self.toLower(retryReceiptEntity.ack().toProtocolTreeNode()) + self.getKeysFor( + [protocolTreeNode["participant"] or protocolTreeNode["from"]], + lambda successJids, b: self.processPlaintextNodeAndSend(messageNode, retryReceiptEntity) if len(successJids) == 1 else None + ) else: #not interested in any non retry receipts, bubble upwards self.toUpper(protocolTreeNode) - def processPlaintextNodeAndSend(self, node): + def processPlaintextNodeAndSend(self, node, retryReceiptEntity = None): recipient_id = node["to"].split('@')[0] isGroup = "-" in recipient_id if node.getChild("media"): self.toLower(node) # skip media enc for now, groups and non groups elif isGroup: - self.sendToGroup(node) + self.sendToGroup(node, retryReceiptEntity) elif self.store.containsSession(recipient_id, 1): self.sendToContact(node) else: - self.getKeysFor([node["to"]], lambda: self.sendToContact(node), lambda: self.toLower(node)) + self.getKeysFor([node["to"]], lambda successJids, b: self.sendToContact(node) if len(successJids) == 1 else self.toLower(node), lambda: self.toLower(node)) @@ -141,14 +150,14 @@ def sendEncEntities(self, node, encEntities): def sendToContact(self, node): recipient_id = node["to"].split('@')[0] cipher = self.getSessionCipher(recipient_id) - messageData = self.serializeToProtobuf(node) + self.getPadding() + messageData = self.serializeToProtobuf(node).SerializeToString() + self.getPadding() ciphertext = cipher.encrypt(messageData) mediaType = node.getChild("media")["type"] if node.getChild("media") else None return self.sendEncEntities(node, [EncProtocolEntity(EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG, 2, ciphertext.serialize(), mediaType)]) - def sendToGroupWithSessions(self, node, jidsNeedSenderKey = None): + def sendToGroupWithSessions(self, node, jidsNeedSenderKey = None, retryCount=0): jidsNeedSenderKey = jidsNeedSenderKey or [] groupJid = node["to"] ownNumber = self.getLayerInterface(YowAuthenticationProtocolLayer).getUsername(False) @@ -159,8 +168,12 @@ def sendToGroupWithSessions(self, node, jidsNeedSenderKey = None): senderKeyDistributionMessage = self.groupSessionBuilder.create(senderKeyName) for jid in jidsNeedSenderKey: sessionCipher = self.getSessionCipher(jid.split('@')[0]) - serialized = self.serializeSenderKeyDistributionMessageToProtobuf(node["to"], senderKeyDistributionMessage) - ciphertext = sessionCipher.encrypt(serialized + self.getPadding()) + message = self.serializeSenderKeyDistributionMessageToProtobuf(node["to"], senderKeyDistributionMessage) + + if retryCount > 0: + message = self.serializeToProtobuf(node, message) + + ciphertext = sessionCipher.encrypt(message.SerializeToString() + self.getPadding()) encEntities.append( EncProtocolEntity( EncProtocolEntity.TYPE_MSG if ciphertext.__class__ == WhisperMessage else EncProtocolEntity.TYPE_PKMSG @@ -168,11 +181,12 @@ def sendToGroupWithSessions(self, node, jidsNeedSenderKey = None): ) ) - messageData = self.serializeToProtobuf(node) - ciphertext = cipher.encrypt(messageData + self.getPadding()) - mediaType = node.getChild("media")["type"] if node.getChild("media") else None + if not retryCount: + messageData = self.serializeToProtobuf(node).SerializeToString() + ciphertext = cipher.encrypt(messageData + self.getPadding()) + mediaType = node.getChild("media")["type"] if node.getChild("media") else None - encEntities.append(EncProtocolEntity(EncProtocolEntity.TYPE_SKMSG, 2, ciphertext, mediaType)) + encEntities.append(EncProtocolEntity(EncProtocolEntity.TYPE_SKMSG, 2, ciphertext, mediaType)) self.sendEncEntities(node, encEntities) @@ -183,11 +197,11 @@ def ensureSessionsAndSendToGroup(self, node, jids): jidsNoSession.append(jid) if len(jidsNoSession): - self.getKeysFor(jidsNoSession, lambda: self.sendToGroupWithSessions(node, jids)) + self.getKeysFor(jidsNoSession, lambda successJids, b: self.sendToGroupWithSessions(node, successJids)) else: self.sendToGroupWithSessions(node, jids) - def sendToGroup(self, node): + def sendToGroup(self, node, retryReceiptEntity = None): groupJid = node["to"] ownNumber = self.getLayerInterface(YowAuthenticationProtocolLayer).getUsername(False) ownJid = self.getLayerInterface(YowAuthenticationProtocolLayer).getUsername(True) @@ -198,39 +212,45 @@ def sendToGroup(self, node): def sendToGroup(resultNode, requestEntity): groupInfo = InfoGroupsResultIqProtocolEntity.fromProtocolTreeNode(resultNode) jids = groupInfo.getParticipants().keys() - return self.ensureSessionsAndSendToGroup(node, [jid for jid in jids if jid != ownJid]) + jids.remove(ownJid) + return self.ensureSessionsAndSendToGroup(node, jids) if senderKeyRecord.isEmpty(): groupInfoIq = InfoGroupsIqProtocolEntity(groupJid) self._sendIq(groupInfoIq, sendToGroup) else: - self.sendToGroupWithSessions(node) - - def serializeToProtobuf(self, node): + retryCount = 0 + jidsNeedSenderKey = [] + if retryReceiptEntity is not None: + retryCount = retryReceiptEntity.getRetryCount() + jidsNeedSenderKey.append(retryReceiptEntity.getRetryJid()) + self.sendToGroupWithSessions(node, jidsNeedSenderKey, retryCount) + + def serializeToProtobuf(self, node, message = None): if node.getChild("body"): - return self.serializeTextToProtobuf(node) + return self.serializeTextToProtobuf(node, message) elif node.getChild("media"): - return self.serializeMediaToProtobuf(node.getChild("media")) + return self.serializeMediaToProtobuf(node.getChild("media"), message) else: raise ValueError("No body or media nodes found") - def serializeTextToProtobuf(self, node): - m = Message() + def serializeTextToProtobuf(self, node, message = None): + m = message or Message() m.conversation = node.getChild("body").getData() - return m.SerializeToString() + return m - def serializeMediaToProtobuf(self, mediaNode): + def serializeMediaToProtobuf(self, mediaNode, message = None): if mediaNode["type"] == "image": - return self.serializeImageToProtobuf(mediaNode) + return self.serializeImageToProtobuf(mediaNode, message) if mediaNode["type"] == "location": - return self.serializeLocationToProtobuf(mediaNode) + return self.serializeLocationToProtobuf(mediaNode, message) if mediaNode["type"] == "vcard": - return self.serializeContactToProtobuf(mediaNode) + return self.serializeContactToProtobuf(mediaNode, message) return None - def serializeLocationToProtobuf(self, mediaNode): - m = Message() + def serializeLocationToProtobuf(self, mediaNode, message = None): + m = message or Message() location_message = LocationMessage() location_message.degress_latitude = float(mediaNode["latitude"]) location_message.degress_longitude = float(mediaNode["longitude"]) @@ -239,20 +259,21 @@ def serializeLocationToProtobuf(self, mediaNode): location_message.url = mediaNode["url"] m.location_message.MergeFrom(location_message) - return m.SerializeToString() - def serializeContactToProtobuf(self, mediaNode): + return m + + def serializeContactToProtobuf(self, mediaNode, message = None): vcardNode = mediaNode.getChild("vcard") - m = Message() + m = message or Message() contact_message = ContactMessage() contact_message.display_name = vcardNode["name"] m.vcard = vcardNode.getData() m.contact_message.MergeFrom(contact_message) - return m.SerializeToString() + return m - def serializeImageToProtobuf(self, mediaNode): - m = Message() + def serializeImageToProtobuf(self, mediaNode, message = None): + m = message or Message() image_message = ImageMessage() image_message.url = mediaNode["url"] image_message.width = int(mediaNode["width"]) @@ -264,22 +285,23 @@ def serializeImageToProtobuf(self, mediaNode): image_message.jpeg_thumbnail = mediaNode.getData() m.image_message.MergeFrom(image_message) - return m.SerializeToString() - def serializeUrlToProtobuf(self, node): + return m + + def serializeUrlToProtobuf(self, node, message = None): pass - def serializeDocumentToProtobuf(self, node): + def serializeDocumentToProtobuf(self, node, message = None): pass - def serializeSenderKeyDistributionMessageToProtobuf(self, groupId, senderKeyDistributionMessage): - m = Message() + def serializeSenderKeyDistributionMessageToProtobuf(self, groupId, senderKeyDistributionMessage, message = None): + m = message or Message() sender_key_distribution_message = ProtoSenderKeyDistributionMessage() sender_key_distribution_message.groupId = groupId sender_key_distribution_message.axolotl_sender_key_distribution_message = senderKeyDistributionMessage.serialize() m.sender_key_distribution_message.MergeFrom(sender_key_distribution_message) # m.conversation = text - return m.SerializeToString() + return m ### def getSessionCipher(self, recipientId): diff --git a/yowsup/layers/axolotl/protocolentities/__init__.py b/yowsup/layers/axolotl/protocolentities/__init__.py index 8aee2456a..0b2a4d28f 100644 --- a/yowsup/layers/axolotl/protocolentities/__init__.py +++ b/yowsup/layers/axolotl/protocolentities/__init__.py @@ -5,3 +5,4 @@ from .enc import EncProtocolEntity from .notification_encrypt import EncryptNotification from .receipt_outgoing_retry import RetryOutgoingReceiptProtocolEntity +from .receipt_incoming_retry import RetryIncomingReceiptProtocolEntity diff --git a/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py b/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py index d243efaa3..69ee66623 100644 --- a/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py +++ b/yowsup/layers/axolotl/protocolentities/receipt_outgoing_retry.py @@ -1,4 +1,4 @@ -from yowsup.structs import ProtocolEntity, ProtocolTreeNode +from yowsup.structs import ProtocolTreeNode from yowsup.layers.protocol_receipts.protocolentities import OutgoingReceiptProtocolEntity from yowsup.layers.axolotl.protocolentities.iq_keys_get_result import ResultGetKeysIqProtocolEntity class RetryOutgoingReceiptProtocolEntity(OutgoingReceiptProtocolEntity): @@ -11,32 +11,35 @@ class RetryOutgoingReceiptProtocolEntity(OutgoingReceiptProtocolEntity): HEX:xxxxxxxxx - ''' - def __init__(self, _id, to, t, v = "1", count = "1",regData = "", participant = None): - super(RetryOutgoingReceiptProtocolEntity, self).__init__(_id,to, participant=participant) - self.setRetryData(t,v,count,regData) - - def setRetryData(self, t,v,count,regData): - self.t = int(t) - self.v = int(v) - self.count = int(count) - self.regData = regData - - def setRegData(self,regData): + def __init__(self, _id, jid, localRegistrationId, retryTimestamp, v = 1, count = 1, participant = None): + super(RetryOutgoingReceiptProtocolEntity, self).__init__(_id, jid, participant=participant) ''' - In axolotl layer: - regData = self.store.getLocalRegistrationId() + Note to self: Android clients won't retry sending if the retry node didn't contain the message timestamp ''' - self.regData = ResultGetKeysIqProtocolEntity._intToBytes(regData) + self.setRetryData(localRegistrationId, v,count, retryTimestamp) + + def setRetryData(self, localRegistrationId, v, count, retryTimestamp): + self.localRegistrationId = localRegistrationId + self.v = v + self.count = count + self.retryTimestamp = int(retryTimestamp) def toProtocolTreeNode(self): node = super(RetryOutgoingReceiptProtocolEntity, self).toProtocolTreeNode() node.setAttribute("type", "retry") - retry = ProtocolTreeNode("retry", {"count": str(self.count),"t":str(self.t),"id":self.getId(),"v":str(self.v)}) + + retryAttribs = { + "count": str(self.count), + "id":self.getId(), + "v":str(self.v), + "t": str(self.retryTimestamp) + } + + retry = ProtocolTreeNode("retry", retryAttribs) node.addChild(retry) - registration = ProtocolTreeNode("registration",data=self.regData) + registration = ProtocolTreeNode("registration", data=ResultGetKeysIqProtocolEntity._intToBytes(self.localRegistrationId)) node.addChild(registration) return node @@ -49,15 +52,16 @@ def fromProtocolTreeNode(node): entity = OutgoingReceiptProtocolEntity.fromProtocolTreeNode(node) entity.__class__ = RetryOutgoingReceiptProtocolEntity retryNode = node.getChild("retry") - entity.setRetryData(retryNode["t"], retryNode["v"], retryNode["count"], node.getChild("registration").data) + entity.setRetryData(ResultGetKeysIqProtocolEntity._bytesToInt(node.getChild("registration").data), retryNode["v"], retryNode["count"], retryNode["t"]) + return entity @staticmethod - def fromMessageNode(MessageNodeToBeRetried): + def fromMessageNode(messageNodeToBeRetried, localRegistrationId): return RetryOutgoingReceiptProtocolEntity( - MessageNodeToBeRetried.getAttributeValue("id"), - MessageNodeToBeRetried.getAttributeValue("from"), - MessageNodeToBeRetried.getAttributeValue("t"), - 1, - participant=MessageNodeToBeRetried.getAttributeValue("participant") + messageNodeToBeRetried.getAttributeValue("id"), + messageNodeToBeRetried.getAttributeValue("from"), + localRegistrationId, + messageNodeToBeRetried.getAttributeValue("t"), + participant=messageNodeToBeRetried.getAttributeValue("participant") ) diff --git a/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py b/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py index e62e4bc68..bb5b8f036 100644 --- a/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py +++ b/yowsup/layers/protocol_acks/protocolentities/ack_outgoing.py @@ -1,4 +1,3 @@ -from yowsup.structs import ProtocolEntity, ProtocolTreeNode from .ack import AckProtocolEntity class OutgoingAckProtocolEntity(AckProtocolEntity): diff --git a/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py b/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py index 181f9b24a..83e22d232 100644 --- a/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py +++ b/yowsup/layers/protocol_receipts/protocolentities/receipt_incoming.py @@ -46,8 +46,9 @@ def __init__(self, _id, _from, timestamp, offline = None, type = None, participa def getType(self): return self.type - def getParticipant(self): - return self.participant + def getParticipant(self, full=True): + if self.participant: + return self.participant if full else self.participant.split('@')[0] def getFrom(self, full = True): return self._from if full else self._from.split('@')[0] From cdbd6e45e37bb3ad9576fb7ea1d872d3fc55e6eb Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 18:44:18 +0200 Subject: [PATCH 51/68] Fixed py3 compat --- setup.py | 2 +- yowsup/layers/axolotl/layer_receive.py | 10 +++++++--- yowsup/layers/axolotl/layer_send.py | 2 +- .../axolotl/protocolentities/receipt_incoming_retry.py | 6 +++--- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 2e2fd6292..bc283d60d 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import platform import sys -deps = ['python-dateutil', 'argparse', 'python-axolotl>=0.1.7'] +deps = ['python-dateutil', 'argparse', 'python-axolotl>=0.1.35', 'six'] if sys.version_info < (2,7): deps += ['importlib'] diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index 3e2c5ad0a..c9c910ad2 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -132,7 +132,8 @@ def handlePreKeyWhisperMessage(self, node): sessionCipher = self.getSessionCipher(pkMessageProtocolEntity.getAuthor(False)) plaintext = sessionCipher.decryptPkmsg(preKeyWhisperMessage) if enc.getVersion() == 2: - padding = ord(plaintext[-1]) & 0xFF + paddingByte = plaintext[-1] if type(plaintext[-1]) is int else ord(plaintext[-1]) + padding = paddingByte & 0xFF self.parseAndHandleMessageProto(pkMessageProtocolEntity, plaintext[:-padding]) else: self.handleConversationMessage(node, plaintext) @@ -146,7 +147,8 @@ def handleWhisperMessage(self, node): plaintext = sessionCipher.decryptMsg(whisperMessage) if enc.getVersion() == 2: - padding = ord(plaintext[-1]) & 0xFF + paddingByte = plaintext[-1] if type(plaintext[-1]) is int else ord(plaintext[-1]) + padding = paddingByte & 0xFF self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) else: self.handleConversationMessage(encMessageProtocolEntity.toProtocolTreeNode(), plaintext) @@ -160,7 +162,9 @@ def handleSenderKeyMessage(self, node): try: plaintext = groupCipher.decrypt(enc.getData()) padding = ord(plaintext[-1]) & 0xFF - self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext[:-padding]) + plaintext = plaintext[:-padding] + plaintext = plaintext.encode() if sys.version_info >= (3, 0) else plaintext + self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext) except NoSessionException as e: logger.error(e) diff --git a/yowsup/layers/axolotl/layer_send.py b/yowsup/layers/axolotl/layer_send.py index f5937bd8b..bbeacd8a6 100644 --- a/yowsup/layers/axolotl/layer_send.py +++ b/yowsup/layers/axolotl/layer_send.py @@ -211,7 +211,7 @@ def sendToGroup(self, node, retryReceiptEntity = None): def sendToGroup(resultNode, requestEntity): groupInfo = InfoGroupsResultIqProtocolEntity.fromProtocolTreeNode(resultNode) - jids = groupInfo.getParticipants().keys() + jids = list(groupInfo.getParticipants().keys()) #keys in py3 returns dict_keys jids.remove(ownJid) return self.ensureSessionsAndSendToGroup(node, jids) diff --git a/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py b/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py index ede7adfeb..9c866853f 100644 --- a/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py +++ b/yowsup/layers/axolotl/protocolentities/receipt_incoming_retry.py @@ -19,9 +19,9 @@ def __init__(self, _id, jid, remoteRegistrationId, receiptTimestamp, retryTimest def setRetryData(self, remoteRegistrationId, v, count, retryTimestamp): self.remoteRegistrationId = remoteRegistrationId - self.v = v - self.count = count - self.retryTimestamp = retryTimestamp + self.v = int(v) + self.count = int(count) + self.retryTimestamp = int(retryTimestamp) def toProtocolTreeNode(self): node = super(RetryIncomingReceiptProtocolEntity, self).toProtocolTreeNode() From 25c99f02dfcfe236cc336e3b92a349e448b7729d Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 19:07:08 +0200 Subject: [PATCH 52/68] Fixed tests for wa1.6 changes --- yowsup/layers/auth/test_authenticator.py | 2 +- yowsup/layers/coder/test_decoder.py | 4 ++-- yowsup/layers/coder/test_encoder.py | 9 +++++---- yowsup/layers/coder/test_tokendictionary.py | 10 +++++----- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/yowsup/layers/auth/test_authenticator.py b/yowsup/layers/auth/test_authenticator.py index bf6ecd8e2..9e6c9a7fc 100644 --- a/yowsup/layers/auth/test_authenticator.py +++ b/yowsup/layers/auth/test_authenticator.py @@ -20,7 +20,7 @@ def setUp(self): def test_streamfeatures(self): self._sendFeatures() - self.assertEqual(self.lowerSink.pop(), StreamFeaturesProtocolEntity(["readreceipts","groups_v2","privacy","presence"]).toProtocolTreeNode()) + self.assertEqual(self.lowerSink.pop(), StreamFeaturesProtocolEntity([]).toProtocolTreeNode()) def test_auth(self): self._sendAuth() diff --git a/yowsup/layers/coder/test_decoder.py b/yowsup/layers/coder/test_decoder.py index c0d4d5778..0ba43acd1 100644 --- a/yowsup/layers/coder/test_decoder.py +++ b/yowsup/layers/coder/test_decoder.py @@ -8,8 +8,8 @@ def setUp(self): self.decoder.streamStarted = True def test_decode(self): - data = [248, 6, 89, 165, 252, 3, 120, 121, 122, 252, 4, 102, 111, 114, 109, 252, 3, 97, 98, 99, 248, 1, - 248, 4, 87, 236, 99, 252, 3, 49, 50, 51, 252, 6, 49, 50, 51, 52, 53, 54] + data = [248, 6, 95, 179, 252, 3, 120, 121, 122, 252, 4, 102, 111, 114, 109, 252, 3, 97, 98, 99, 248, 1, 248, 4, 93, + 236, 104, 255, 130, 18, 63, 252, 6, 49, 50, 51, 52, 53, 54] node = self.decoder.getProtocolTreeNode(data) targetNode = ProtocolTreeNode("message", {"form": "abc", "to":"xyz"}, [ProtocolTreeNode("media", {"width" : "123"}, data="123456")]) self.assertEqual(node, targetNode) diff --git a/yowsup/layers/coder/test_encoder.py b/yowsup/layers/coder/test_encoder.py index 2ac34214e..99c8f13e6 100644 --- a/yowsup/layers/coder/test_encoder.py +++ b/yowsup/layers/coder/test_encoder.py @@ -13,9 +13,10 @@ def test_encode(self): result = self.encoder.protocolTreeNodeToBytes(node) self.assertTrue(result in ( - [248, 6, 89, 252, 4, 102, 111, 114, 109, 252, 3, 97, 98, 99, 165, 252, 3, 120, 121, 122, 248, 1, - 248, 4, 87, 236, 99, 252, 3, 49, 50, 51, 252, 6, 49, 50, 51, 52, 53, 54], - [248, 6, 89, 165, 252, 3, 120, 121, 122, 252, 4, 102, 111, 114, 109, 252, 3, 97, 98, 99, 248, 1, - 248, 4, 87, 236, 99, 252, 3, 49, 50, 51, 252, 6, 49, 50, 51, 52, 53, 54]) + [248, 6, 95, 179, 252, 3, 120, 121, 122, 252, 4, 102, 111, 114, 109, 252, 3, 97, 98, 99, 248, 1, 248, 4, 93, + 236, 104, 255, 130, 18, 63, 252, 6, 49, 50, 51, 52, 53, 54], + [248, 6, 95, 252, 4, 102, 111, 114, 109, 252, 3, 97, 98, 99, 179, 252, 3, 120, 121, 122, 248, 1, 248, 4, 93, + 236, 104, 255, 130, 18, 63, 252, 6, 49, 50, 51, 52, 53, 54] + ) ) diff --git a/yowsup/layers/coder/test_tokendictionary.py b/yowsup/layers/coder/test_tokendictionary.py index 55b6738dc..7fe40c243 100644 --- a/yowsup/layers/coder/test_tokendictionary.py +++ b/yowsup/layers/coder/test_tokendictionary.py @@ -5,16 +5,16 @@ def setUp(self): self.tokenDictionary = TokenDictionary() def test_getToken(self): - self.assertEqual(self.tokenDictionary.getToken(74), "iq") + self.assertEqual(self.tokenDictionary.getToken(80), "iq") def test_getIndex(self): - self.assertEqual(self.tokenDictionary.getIndex("iq"), (74, False)) + self.assertEqual(self.tokenDictionary.getIndex("iq"), (80, False)) def test_getSecondaryToken(self): - self.assertEqual(self.tokenDictionary.getToken(238), "wmv") + self.assertEqual(self.tokenDictionary.getToken(238), "amrnb") def test_getSecondaryTokenExplicit(self): - self.assertEqual(self.tokenDictionary.getToken(1, True), "wmv") + self.assertEqual(self.tokenDictionary.getToken(11, True), "wmv") def test_getSecondaryIndex(self): - self.assertEqual(self.tokenDictionary.getIndex("wmv"), (1, True)) \ No newline at end of file + self.assertEqual(self.tokenDictionary.getIndex("wmv"), (11, True)) \ No newline at end of file From fb8ba4e1f9708cfe8922d5fd798681c555792ba2 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 19:34:38 +0200 Subject: [PATCH 53/68] py26 fixed error --- yowsup/layers/coder/decoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index e6ffa4f46..7d86557c0 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -50,7 +50,7 @@ def streamStart(self, data): def readNibble(self, data): _byte = self.readInt8(data) ignoreLastNibble = bool(_byte & 0x80) - size = (_byte & 0x7f); + size = (_byte & 0x7f) nrOfNibbles = size * 2 - int(ignoreLastNibble) dataArr = self.readArray(size, data) string = '' @@ -74,7 +74,7 @@ def readPacked8(self, n, data): remove = 1 size = size & 0x7F text = bytearray(self.readArray(size, data)) - hexData = binascii.hexlify(text).upper() + hexData = binascii.hexlify(str(text) if sys.version_info < (2,7) else text).upper() dataSize = len(hexData) out = [] if remove == 0: From 8e814ff8cd072602ba033c33512ec23e072f92c6 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 24 Apr 2016 21:56:44 +0200 Subject: [PATCH 54/68] Fixed python2.6 support Closes #619 Truncated message Closes #1460 YowAxolotlLayer' object has no attribute '_store' Probably Closes #1120 Not able to install yowsup in rhel Fixes #1050 DecodeError: Truncated message Probably Closes #966 Truncated string Probably Closes #619 Truncated message --- yowsup/layers/__init__.py | 1 + yowsup/layers/axolotl/layer_base.py | 21 ++++++++------- .../store/sqlite/liteidentitykeystore.py | 27 ++++++++++++++----- .../axolotl/store/sqlite/liteprekeystore.py | 4 ++- .../store/sqlite/litesenderkeystore.py | 8 ++++-- .../axolotl/store/sqlite/litesessionstore.py | 4 ++- .../store/sqlite/litesignedprekeystore.py | 4 ++- 7 files changed, 49 insertions(+), 20 deletions(-) diff --git a/yowsup/layers/__init__.py b/yowsup/layers/__init__.py index b051c9051..9d9cb9a1b 100644 --- a/yowsup/layers/__init__.py +++ b/yowsup/layers/__init__.py @@ -44,6 +44,7 @@ def __init__(self): self.setLayers(None, None) self.interface = None self.event_callbacks = {} + self.__stack = None members = inspect.getmembers(self, predicate=inspect.ismethod) for m in members: if hasattr(m[1], "event_callback"): diff --git a/yowsup/layers/axolotl/layer_base.py b/yowsup/layers/axolotl/layer_base.py index acbea2359..270685615 100644 --- a/yowsup/layers/axolotl/layer_base.py +++ b/yowsup/layers/axolotl/layer_base.py @@ -14,7 +14,7 @@ class AxolotlBaseLayer(YowProtocolLayer): _DB = "axolotl.db" def __init__(self): super(AxolotlBaseLayer, self).__init__() - self.store = None + self._store = None self.skipEncJids = [] def onNewStoreSet(self, store): @@ -28,15 +28,18 @@ def receive(self, node): @property def store(self): - if self._store is None: - self.store = LiteAxolotlStore( - StorageTools.constructPath( - self.getProp( - YowAuthenticationProtocolLayer.PROP_CREDENTIALS)[0], - self.__class__._DB + try: + if self._store is None: + self.store = LiteAxolotlStore( + StorageTools.constructPath( + self.getProp( + YowAuthenticationProtocolLayer.PROP_CREDENTIALS)[0], + self.__class__._DB + ) ) - ) - return self._store + return self._store + except AttributeError: + return None @store.setter def store(self, store): diff --git a/yowsup/layers/axolotl/store/sqlite/liteidentitykeystore.py b/yowsup/layers/axolotl/store/sqlite/liteidentitykeystore.py index 0d3ebeb92..f3489ac23 100644 --- a/yowsup/layers/axolotl/store/sqlite/liteidentitykeystore.py +++ b/yowsup/layers/axolotl/store/sqlite/liteidentitykeystore.py @@ -1,9 +1,8 @@ from axolotl.state.identitykeystore import IdentityKeyStore -from axolotl.ecc.curve import Curve from axolotl.identitykey import IdentityKey -from axolotl.util.keyhelper import KeyHelper from axolotl.identitykeypair import IdentityKeyPair from axolotl.ecc.djbec import * +import sys class LiteIdentityKeyStore(IdentityKeyStore): @@ -42,8 +41,16 @@ def getLocalRegistrationId(self): def storeLocalData(self, registrationId, identityKeyPair): q = "INSERT INTO identities(recipient_id, registration_id, public_key, private_key) VALUES(-1, ?, ?, ?)" c = self.dbConn.cursor() - c.execute(q, (registrationId, identityKeyPair.getPublicKey().getPublicKey().serialize(), - identityKeyPair.getPrivateKey().serialize())) + pubKey = identityKeyPair.getPublicKey().getPublicKey().serialize() + privKey = identityKeyPair.getPrivateKey().serialize() + + if sys.version_info < (2,7): + pubKey = buffer(pubKey) + privKey = buffer(privKey) + + c.execute(q, (registrationId, + pubKey, + privKey)) self.dbConn.commit() @@ -55,7 +62,9 @@ def saveIdentity(self, recipientId, identityKey): q = "INSERT INTO identities (recipient_id, public_key) VALUES(?, ?)" c = self.dbConn.cursor() - c.execute(q, (recipientId, identityKey.getPublicKey().serialize())) + + pubKey = identityKey.getPublicKey().serialize() + c.execute(q, (recipientId, buffer(pubKey) if sys.version_info < (2,7) else pubKey)) self.dbConn.commit() def isTrustedIdentity(self, recipientId, identityKey): @@ -65,4 +74,10 @@ def isTrustedIdentity(self, recipientId, identityKey): result = c.fetchone() if not result: return True - return result[0] == identityKey.getPublicKey().serialize() \ No newline at end of file + + pubKey = identityKey.getPublicKey().serialize() + + if sys.version_info < (2, 7): + pubKey = buffer(pubKey) + + return result[0] == pubKey \ No newline at end of file diff --git a/yowsup/layers/axolotl/store/sqlite/liteprekeystore.py b/yowsup/layers/axolotl/store/sqlite/liteprekeystore.py index 48be4ef1d..1e5acf407 100644 --- a/yowsup/layers/axolotl/store/sqlite/liteprekeystore.py +++ b/yowsup/layers/axolotl/store/sqlite/liteprekeystore.py @@ -1,5 +1,6 @@ from axolotl.state.prekeystore import PreKeyStore from axolotl.state.prekeyrecord import PreKeyRecord +import sys class LitePreKeyStore(PreKeyStore): def __init__(self, dbConn): """ @@ -33,7 +34,8 @@ def storePreKey(self, preKeyId, preKeyRecord): #self.removePreKey(preKeyId) q = "INSERT INTO prekeys (prekey_id, record) VALUES(?,?)" cursor = self.dbConn.cursor() - cursor.execute(q, (preKeyId, preKeyRecord.serialize())) + serialized = preKeyRecord.serialize() + cursor.execute(q, (preKeyId, buffer(serialized) if sys.version_info < (2,7) else serialized)) self.dbConn.commit() def containsPreKey(self, preKeyId): diff --git a/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py b/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py index 751166a69..7c841caac 100644 --- a/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py +++ b/yowsup/layers/axolotl/store/sqlite/litesenderkeystore.py @@ -1,6 +1,7 @@ from axolotl.groups.state.senderkeystore import SenderKeyStore from axolotl.groups.state.senderkeyrecord import SenderKeyRecord import sqlite3 +import sys class LiteSenderKeyStore(SenderKeyStore): def __init__(self, dbConn): """ @@ -20,13 +21,16 @@ def storeSenderKey(self, senderKeyName, senderKeyRecord): """ q = "INSERT INTO sender_keys (group_id, sender_id, record) VALUES(?,?, ?)" cursor = self.dbConn.cursor() + serialized = senderKeyRecord.serialize() + if sys.version_info < (2,7): + serialized = buffer(serialized) try: - cursor.execute(q, (senderKeyName.getGroupId(), senderKeyName.getSender().getName(), senderKeyRecord.serialize())) + cursor.execute(q, (senderKeyName.getGroupId(), senderKeyName.getSender().getName(), serialized)) self.dbConn.commit() except sqlite3.IntegrityError as e: q = "UPDATE sender_keys set record = ? WHERE group_id = ? and sender_id = ?" cursor = self.dbConn.cursor() - cursor.execute(q, (senderKeyRecord.serialize(), senderKeyName.getGroupId(), senderKeyName.getSender().getName())) + cursor.execute(q, (serialized, senderKeyName.getGroupId(), senderKeyName.getSender().getName())) self.dbConn.commit() def loadSenderKey(self, senderKeyName): diff --git a/yowsup/layers/axolotl/store/sqlite/litesessionstore.py b/yowsup/layers/axolotl/store/sqlite/litesessionstore.py index 25bae00b1..e5ea116bd 100644 --- a/yowsup/layers/axolotl/store/sqlite/litesessionstore.py +++ b/yowsup/layers/axolotl/store/sqlite/litesessionstore.py @@ -1,5 +1,6 @@ from axolotl.state.sessionstore import SessionStore from axolotl.state.sessionrecord import SessionRecord +import sys class LiteSessionStore(SessionStore): def __init__(self, dbConn): """ @@ -35,7 +36,8 @@ def storeSession(self, recipientId, deviceId, sessionRecord): q = "INSERT INTO sessions(recipient_id, device_id, record) VALUES(?,?,?)" c = self.dbConn.cursor() - c.execute(q, (recipientId, deviceId, sessionRecord.serialize())) + serialized = sessionRecord.serialize() + c.execute(q, (recipientId, deviceId, buffer(serialized) if sys.version_info < (2,7) else serialized)) self.dbConn.commit() def containsSession(self, recipientId, deviceId): diff --git a/yowsup/layers/axolotl/store/sqlite/litesignedprekeystore.py b/yowsup/layers/axolotl/store/sqlite/litesignedprekeystore.py index 0a977c575..848f4c40e 100644 --- a/yowsup/layers/axolotl/store/sqlite/litesignedprekeystore.py +++ b/yowsup/layers/axolotl/store/sqlite/litesignedprekeystore.py @@ -1,6 +1,7 @@ from axolotl.state.signedprekeystore import SignedPreKeyStore from axolotl.state.signedprekeyrecord import SignedPreKeyRecord from axolotl.invalidkeyidexception import InvalidKeyIdException +import sys class LiteSignedPreKeyStore(SignedPreKeyStore): def __init__(self, dbConn): """ @@ -42,7 +43,8 @@ def storeSignedPreKey(self, signedPreKeyId, signedPreKeyRecord): q = "INSERT INTO signed_prekeys (prekey_id, record) VALUES(?,?)" cursor = self.dbConn.cursor() - cursor.execute(q, (signedPreKeyId, signedPreKeyRecord.serialize())) + record = signedPreKeyRecord.serialize() + cursor.execute(q, (signedPreKeyId, buffer(record) if sys.version_info < (2,7) else record)) self.dbConn.commit() def containsSignedPreKey(self, signedPreKeyId): From cc1fa3c18d7c2abf418571c26d7d7b808ddf7798 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Wed, 27 Apr 2016 12:19:00 +0200 Subject: [PATCH 55/68] Adjusted decryption results log messages --- yowsup/layers/axolotl/layer_receive.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index c9c910ad2..fe5271109 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -79,7 +79,6 @@ def handleEncMessage(self, node): encMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) isGroup = node["participant"] is not None senderJid = node["participant"] if isGroup else node["from"] - encNode = None if node.getChild("enc")["v"] == "2" and node["from"] not in self.v2Jids: self.v2Jids.append(node["from"]) try: @@ -89,17 +88,12 @@ def handleEncMessage(self, node): self.handleWhisperMessage(node) if encMessageProtocolEntity.getEnc(EncProtocolEntity.TYPE_SKMSG): self.handleSenderKeyMessage(node) - except InvalidMessageException as e: - logger.error(e) + except (InvalidMessageException, InvalidKeyIdException) as e: + logger.warning("InvalidMessage or KeyId for %s, going to send a retry", encMessageProtocolEntity.getAuthor(False)) retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node, self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) - except InvalidKeyIdException as e: - logger.error(e) - retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node,self.store.getLocalRegistrationId()) - self.toLower(retry.toProtocolTreeNode()) except NoSessionException as e: - logger.error(e) - entity = GetKeysIqProtocolEntity([senderJid]) + logger.warning("No session for %s, getting their keys now", encMessageProtocolEntity.getAuthor(False)) conversationIdentifier = (node["from"], node["participant"]) @@ -112,18 +106,16 @@ def handleEncMessage(self, node): self.getKeysFor([senderJid], successFn) except DuplicateMessageException as e: - logger.error(e) - logger.warning("Going to send the delivery receipt myself !") + logger.warning("Received a message that we've previously decrypted, goint to send the delivery receipt myself", exc_info=1) self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"], participant=node["participant"]).toProtocolTreeNode()) except UntrustedIdentityException as e: if self.getProp(PROP_IDENTITY_AUTOTRUST, False): - logger.warning("Autotrusting identity for %s" % e.getName()) + logger.warning("Autotrusting identity for %s", e.getName(), exc_info=1) self.store.saveIdentity(e.getName(), e.getIdentityKey()) return self.handleEncMessage(node) else: - logger.error(e) - logger.warning("Ignoring message with untrusted identity") + logger.error("Ignoring message with untrusted identity", exc_info=1) def handlePreKeyWhisperMessage(self, node): pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) @@ -167,7 +159,7 @@ def handleSenderKeyMessage(self, node): self.parseAndHandleMessageProto(encMessageProtocolEntity, plaintext) except NoSessionException as e: - logger.error(e) + logger.warning("No session for %s, going to send a retry", encMessageProtocolEntity.getAuthor(False)) retry = RetryOutgoingReceiptProtocolEntity.fromMessageNode(node, self.store.getLocalRegistrationId()) self.toLower(retry.toProtocolTreeNode()) From 92e802b2bc24c893103e99d6edb829c73d515c4e Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Wed, 27 Apr 2016 12:21:00 +0200 Subject: [PATCH 56/68] removed exc_info from logs --- yowsup/layers/axolotl/layer_receive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index fe5271109..150d88e7c 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -106,16 +106,16 @@ def handleEncMessage(self, node): self.getKeysFor([senderJid], successFn) except DuplicateMessageException as e: - logger.warning("Received a message that we've previously decrypted, goint to send the delivery receipt myself", exc_info=1) + logger.warning("Received a message that we've previously decrypted, goint to send the delivery receipt myself") self.toLower(OutgoingReceiptProtocolEntity(node["id"], node["from"], participant=node["participant"]).toProtocolTreeNode()) except UntrustedIdentityException as e: if self.getProp(PROP_IDENTITY_AUTOTRUST, False): - logger.warning("Autotrusting identity for %s", e.getName(), exc_info=1) + logger.warning("Autotrusting identity for %s", e.getName()) self.store.saveIdentity(e.getName(), e.getIdentityKey()) return self.handleEncMessage(node) else: - logger.error("Ignoring message with untrusted identity", exc_info=1) + logger.error("Ignoring message with untrusted identity") def handlePreKeyWhisperMessage(self, node): pkMessageProtocolEntity = EncryptedMessageProtocolEntity.fromProtocolTreeNode(node) From 91e577624d052e6d5ff445002a4ba3de00a8f665 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 29 Apr 2016 07:13:01 +0200 Subject: [PATCH 57/68] Fixed s40 env --- yowsup/env/env_s40.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yowsup/env/env_s40.py b/yowsup/env/env_s40.py index e92e8714b..6dbe169c2 100644 --- a/yowsup/env/env_s40.py +++ b/yowsup/env/env_s40.py @@ -36,5 +36,6 @@ def getUserAgent(self): WHATSAPP_VERSION = self.getVersion(), OS_NAME = self.getOSName() + "Version", OS_VERSION = self.getOSVersion(), - DEVICE_NAME = self.getDeviceName() + DEVICE_NAME = self.getDeviceName(), + MANUFACTURER = self.getManufacturer() ) From 275c932e6e558ae857635627beceead2242671c3 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 29 Apr 2016 09:53:43 +0200 Subject: [PATCH 58/68] Updated s40 token, set s40 as default env --- yowsup/env/env.py | 11 ++++++++--- yowsup/env/env_s40.py | 5 ++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/yowsup/env/env.py b/yowsup/env/env.py index a3ae0c8b4..c03e8002c 100644 --- a/yowsup/env/env.py +++ b/yowsup/env/env.py @@ -4,6 +4,8 @@ logger = logging.getLogger(__name__) +DEFAULT = "s40" + class YowsupEnvType(abc.ABCMeta): def __init__(cls, name, bases, dct): if name != "YowsupEnv": @@ -44,9 +46,12 @@ def getRegisteredEnvs(cls): @classmethod def getCurrent(cls): if cls.__CURR is None: - newEnvName = cls.getRegisteredEnvs()[0] - logger.debug("Env not set, setting it to %s" % newEnvName ) - cls.setEnv(newEnvName) + env = DEFAULT + envs = cls.getRegisteredEnvs() + if env not in envs: + env = envs[0] + logger.debug("Env not set, setting it to %s" % env) + cls.setEnv(env) return cls.__CURR @abc.abstractmethod diff --git a/yowsup/env/env_s40.py b/yowsup/env/env_s40.py index 6dbe169c2..d1ba6fbbe 100644 --- a/yowsup/env/env_s40.py +++ b/yowsup/env/env_s40.py @@ -1,13 +1,12 @@ from .env import YowsupEnv -import base64 import hashlib class S40YowsupEnv(YowsupEnv): - _VERSION = "2.13.39" + _VERSION = "2.16.6" _OS_NAME= "S40" _OS_VERSION = "14.26" _DEVICE_NAME = "302" _MANUFACTURER = "Nokia" - _TOKEN_STRING = "PdA2DJyKoUrwLw1Bg6EIhzh502dF9noR9uFCllGk1456529096701{phone}" + _TOKEN_STRING = "PdA2DJyKoUrwLw1Bg6EIhzh502dF9noR9uFCllGk1461880318866{phone}" _AXOLOTL = True def getVersion(self): From cf214af4787fa7296bbe5f61ab24494a7ff3d54f Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Fri, 29 Apr 2016 12:21:26 +0200 Subject: [PATCH 59/68] Fixed utcTimestamp returning local timestamp --- yowsup/common/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yowsup/common/tools.py b/yowsup/common/tools.py index 7a83919eb..6bf3e1094 100644 --- a/yowsup/common/tools.py +++ b/yowsup/common/tools.py @@ -1,4 +1,5 @@ import time,datetime,re, hashlib +import calendar from dateutil import tz import os from .constants import YowConstants @@ -108,9 +109,8 @@ def utcToLocal(dt): @staticmethod def utcTimestamp(): - #utc = tz.gettz('UTC') utcNow = datetime.datetime.utcnow() - return TimeTools.datetimeToTimestamp(utcNow) + return calendar.timegm(utcNow.timetuple()) @staticmethod def datetimeToTimestamp(dt): From fd218791aee939737f037540d72f3b2979550a0c Mon Sep 17 00:00:00 2001 From: rigid Date: Fri, 29 Apr 2016 14:57:29 +0200 Subject: [PATCH 60/68] encode hash as 'hex' so output is always printable --- .../protocolentities/message_media_downloadable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py index 7286fdab9..2694fa344 100644 --- a/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py +++ b/yowsup/layers/protocol_media/protocolentities/message_media_downloadable.py @@ -29,7 +29,7 @@ def __init__(self, mediaType, def __str__(self): out = super(DownloadableMediaMessageProtocolEntity, self).__str__() out += "MimeType: %s\n" % self.mimeType - out += "File Hash: %s\n" % self.fileHash + out += "File Hash: %s\n" % self.fileHash.encode('hex') out += "URL: %s\n" % self.url out += "IP: %s\n" % self.ip out += "File Size: %s\n" % self.size From e3098214e08a58fa810223056cb3f33beceaaaa1 Mon Sep 17 00:00:00 2001 From: Jlguardi Date: Mon, 16 May 2016 01:11:04 +0200 Subject: [PATCH 61/68] Fixed #1578 --- yowsup/common/tools.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/yowsup/common/tools.py b/yowsup/common/tools.py index dbeda2d6f..cbbab33ab 100644 --- a/yowsup/common/tools.py +++ b/yowsup/common/tools.py @@ -152,8 +152,10 @@ def generatePreviewFromImage(image): class MimeTools: MIME_FILE = os.path.join(os.path.dirname(__file__), 'mime.types') mimetypes.init() # Load default mime.types - mimetypes.init([MIME_FILE]) # Append whatsapp mime.types - decode_hex = codecs.getdecoder("hex_codec") + try: + mimetypes.init([MIME_FILE]) # Append whatsapp mime.types + except exception as e: + logger.warning("Mime types supported can't be read. System mimes will be used. Cause: " + e.message) @staticmethod def getMIME(filepath): From 8efe749f089bc872a8001d668cb681e57174e0c2 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 12:56:17 +0200 Subject: [PATCH 62/68] Detect blocked in exists result, during code request --- yowsup/registration/coderequest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yowsup/registration/coderequest.py b/yowsup/registration/coderequest.py index 5d76187a9..d0c5d91eb 100644 --- a/yowsup/registration/coderequest.py +++ b/yowsup/registration/coderequest.py @@ -53,6 +53,8 @@ def send(self, parser = None): result = request.send() if result["status"] == "ok": return result + elif result["status"] == "fail" and "reason" in result and result["reason"] == "blocked": + return result self.__id = WATools.generateIdentity() self.addParam("id", self.__id) From d2f17664c824112c59fc9aa0e724f30479de2742 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 13:48:53 +0200 Subject: [PATCH 63/68] Fixed typo, closes #1580 --- yowsup/layers/axolotl/layer_receive.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yowsup/layers/axolotl/layer_receive.py b/yowsup/layers/axolotl/layer_receive.py index 150d88e7c..5ec329307 100644 --- a/yowsup/layers/axolotl/layer_receive.py +++ b/yowsup/layers/axolotl/layer_receive.py @@ -245,7 +245,7 @@ def handleDocumentMessage(self, originalEncNode, documentMessage): pass def handleLocationMessage(self, originalEncNode, locationMessage): - messageNode = copy.deepycopy(originalEncNode) + messageNode = copy.deepcopy(originalEncNode) messageNode["type"] = "media" mediaNode = ProtocolTreeNode("media", { "latitude": locationMessage.degrees_latitude, @@ -259,7 +259,7 @@ def handleLocationMessage(self, originalEncNode, locationMessage): self.toUpper(messageNode) def handleContactMessage(self, originalEncNode, contactMessage): - messageNode = copy.deepycopy(originalEncNode) + messageNode = copy.deepcopy(originalEncNode) messageNode["type"] = "media" mediaNode = ProtocolTreeNode("media", { "type": "vcard" From 8af0f399429324fe90c5e6528d8d30a11f75366e Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 14:18:32 +0200 Subject: [PATCH 64/68] updated s40 env to 2.16.7 --- yowsup/env/env_s40.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yowsup/env/env_s40.py b/yowsup/env/env_s40.py index d1ba6fbbe..49e7c73e1 100644 --- a/yowsup/env/env_s40.py +++ b/yowsup/env/env_s40.py @@ -1,12 +1,12 @@ from .env import YowsupEnv import hashlib class S40YowsupEnv(YowsupEnv): - _VERSION = "2.16.6" + _VERSION = "2.16.7" _OS_NAME= "S40" _OS_VERSION = "14.26" _DEVICE_NAME = "302" _MANUFACTURER = "Nokia" - _TOKEN_STRING = "PdA2DJyKoUrwLw1Bg6EIhzh502dF9noR9uFCllGk1461880318866{phone}" + _TOKEN_STRING = "PdA2DJyKoUrwLw1Bg6EIhzh502dF9noR9uFCllGk1462212402694{phone}" _AXOLOTL = True def getVersion(self): From ac2dd59bb65eb6e18343400506beafdc34a78e44 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 14:19:44 +0200 Subject: [PATCH 65/68] Fixed error in printing stream:error data --- yowsup/layers/auth/protocolentities/stream_error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yowsup/layers/auth/protocolentities/stream_error.py b/yowsup/layers/auth/protocolentities/stream_error.py index c44bec387..10882e8dd 100644 --- a/yowsup/layers/auth/protocolentities/stream_error.py +++ b/yowsup/layers/auth/protocolentities/stream_error.py @@ -43,7 +43,7 @@ def getErrorType(self): def __str__(self): out = "Stream Error type: %s\n" % self.getErrorType() - out += self.getErrorData() + out += "%s" % self.getErrorData() out += "\n" return out From ab0133ada03ae2b0ef6288b282340daea14f3e81 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 14:20:08 +0200 Subject: [PATCH 66/68] Fixed error when node data is string, closes #1530 --- yowsup/layers/coder/decoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yowsup/layers/coder/decoder.py b/yowsup/layers/coder/decoder.py index 7d86557c0..652662d47 100644 --- a/yowsup/layers/coder/decoder.py +++ b/yowsup/layers/coder/decoder.py @@ -239,7 +239,7 @@ def nextTreeInternal(self, data): if size == 0 or tag is None: raise ValueError("nextTree sees 0 list or null tag") - attribCount = (size - 2 + size % 2)/2; + attribCount = (size - 2 + size % 2)/2 attribs = self.readAttributes(attribCount, data) if size % 2 ==1: return ProtocolTreeNode(tag, attribs) @@ -264,7 +264,7 @@ def nextTreeInternal(self, data): else: nodeData = self.readString(read2, data) - if nodeData: + if nodeData and type(nodeData) is not str: nodeData = "".join(map(chr, nodeData)) return ProtocolTreeNode(tag, attribs, nodeChildren, nodeData) From 47dbdc8688c88121c55a1be6faf592b280ab5254 Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 14:21:25 +0200 Subject: [PATCH 67/68] Don't autoreconnect when signed in in another location --- yowsup/layers/interface/interface.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/yowsup/layers/interface/interface.py b/yowsup/layers/interface/interface.py index 8f3ef0bc2..114cab3b9 100644 --- a/yowsup/layers/interface/interface.py +++ b/yowsup/layers/interface/interface.py @@ -1,10 +1,10 @@ from yowsup.layers import YowLayer, YowLayerEvent from yowsup.layers.protocol_iq.protocolentities import IqProtocolEntity -from yowsup.layers.network import YowNetworkLayer from yowsup.layers.auth import YowAuthenticationProtocolLayer from yowsup.layers.protocol_media.protocolentities.iq_requestupload import RequestUploadIqProtocolEntity from yowsup.layers.protocol_media.mediauploader import MediaUploader from yowsup.layers.network.layer import YowNetworkLayer +from yowsup.layers.auth.protocolentities import StreamErrorProtocolEntity from yowsup.layers import EventCallback import inspect import logging @@ -85,12 +85,15 @@ def receive(self, entity): def onStreamError(self, streamErrorEntity): logger.error(streamErrorEntity) if self.getProp(self.__class__.PROP_RECONNECT_ON_STREAM_ERR, True): - logger.info("Initiating reconnect") - self.reconnect = True - self.disconnect() + if streamErrorEntity.getErrorType() == StreamErrorProtocolEntity.TYPE_CONFLICT: + logger.warn("Not reconnecting because you signed in in another location") + else: + logger.info("Initiating reconnect") + self.reconnect = True else: - logger.warn("No reconnecting because property %s is not set" % self.__class__.PROP_RECONNECT_ON_STREAM_ERR) + logger.warn("Not reconnecting because property %s is not set" % self.__class__.PROP_RECONNECT_ON_STREAM_ERR) self.toUpper(streamErrorEntity) + self.disconnect() @EventCallback(YowNetworkLayer.EVENT_STATE_CONNECTED) def onConnected(self, yowLayerEvent): From 5d1f26a692441854caa36ff7243c729484e8841b Mon Sep 17 00:00:00 2001 From: Tarek Galal Date: Sun, 22 May 2016 14:48:30 +0200 Subject: [PATCH 68/68] bumped to 2.5.0 --- README.md | 4 ++-- yowsup/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8a2ee169b..960ecdc7b 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ -## Updates (April 1, 2016) -Yowsup v2.4.102 is out, See [release notes](https://github.com/tgalal/yowsup/releases/tag/v2.4.102) +## Updates (May 22, 2016) +Yowsup v2.5.0 is out, See [release notes](https://github.com/tgalal/yowsup/releases/tag/v2.5.0) ========================================================== diff --git a/yowsup/__init__.py b/yowsup/__init__.py index 255448118..77f582639 100644 --- a/yowsup/__init__.py +++ b/yowsup/__init__.py @@ -1,2 +1,2 @@ -__version__ = "2.4.103" +__version__ = "2.5.0" __author__ = "Tarek Galal"