1 //===-- PythonDataObjects.cpp ---------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "lldb/Host/Config.h" 10 11 #if LLDB_ENABLE_PYTHON 12 13 #include "PythonDataObjects.h" 14 #include "ScriptInterpreterPython.h" 15 16 #include "lldb/Host/File.h" 17 #include "lldb/Host/FileSystem.h" 18 #include "lldb/Interpreter/ScriptInterpreter.h" 19 #include "lldb/Utility/LLDBLog.h" 20 #include "lldb/Utility/Log.h" 21 #include "lldb/Utility/Stream.h" 22 23 #include "llvm/Support/Casting.h" 24 #include "llvm/Support/ConvertUTF.h" 25 #include "llvm/Support/Errno.h" 26 27 #include <cstdio> 28 #include <variant> 29 30 using namespace lldb_private; 31 using namespace lldb; 32 using namespace lldb_private::python; 33 using llvm::cantFail; 34 using llvm::Error; 35 using llvm::Expected; 36 using llvm::Twine; 37 As(Expected<PythonObject> && obj)38 template <> Expected<bool> python::As<bool>(Expected<PythonObject> &&obj) { 39 if (!obj) 40 return obj.takeError(); 41 return obj.get().IsTrue(); 42 } 43 44 template <> As(Expected<PythonObject> && obj)45 Expected<long long> python::As<long long>(Expected<PythonObject> &&obj) { 46 if (!obj) 47 return obj.takeError(); 48 return obj->AsLongLong(); 49 } 50 51 template <> 52 Expected<unsigned long long> As(Expected<PythonObject> && obj)53 python::As<unsigned long long>(Expected<PythonObject> &&obj) { 54 if (!obj) 55 return obj.takeError(); 56 return obj->AsUnsignedLongLong(); 57 } 58 59 template <> As(Expected<PythonObject> && obj)60 Expected<std::string> python::As<std::string>(Expected<PythonObject> &&obj) { 61 if (!obj) 62 return obj.takeError(); 63 PyObject *str_obj = PyObject_Str(obj.get().get()); 64 if (!str_obj) 65 return llvm::make_error<PythonException>(); 66 auto str = Take<PythonString>(str_obj); 67 auto utf8 = str.AsUTF8(); 68 if (!utf8) 69 return utf8.takeError(); 70 return std::string(utf8.get()); 71 } 72 python_is_finalizing()73 static bool python_is_finalizing() { 74 #if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 13) || (PY_MAJOR_VERSION > 3) 75 return Py_IsFinalizing(); 76 #elif PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 7 77 return _Py_Finalizing != nullptr; 78 #else 79 return _Py_IsFinalizing(); 80 #endif 81 } 82 Reset()83 void PythonObject::Reset() { 84 if (m_py_obj && Py_IsInitialized()) { 85 if (python_is_finalizing()) { 86 // Leak m_py_obj rather than crashing the process. 87 // https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure 88 } else { 89 PyGILState_STATE state = PyGILState_Ensure(); 90 Py_DECREF(m_py_obj); 91 PyGILState_Release(state); 92 } 93 } 94 m_py_obj = nullptr; 95 } 96 AsLongLong() const97 Expected<long long> PythonObject::AsLongLong() const { 98 if (!m_py_obj) 99 return nullDeref(); 100 assert(!PyErr_Occurred()); 101 long long r = PyLong_AsLongLong(m_py_obj); 102 if (PyErr_Occurred()) 103 return exception(); 104 return r; 105 } 106 AsUnsignedLongLong() const107 Expected<unsigned long long> PythonObject::AsUnsignedLongLong() const { 108 if (!m_py_obj) 109 return nullDeref(); 110 assert(!PyErr_Occurred()); 111 long long r = PyLong_AsUnsignedLongLong(m_py_obj); 112 if (PyErr_Occurred()) 113 return exception(); 114 return r; 115 } 116 117 // wraps on overflow, instead of raising an error. AsModuloUnsignedLongLong() const118 Expected<unsigned long long> PythonObject::AsModuloUnsignedLongLong() const { 119 if (!m_py_obj) 120 return nullDeref(); 121 assert(!PyErr_Occurred()); 122 unsigned long long r = PyLong_AsUnsignedLongLongMask(m_py_obj); 123 // FIXME: We should fetch the exception message and hoist it. 124 if (PyErr_Occurred()) 125 return exception(); 126 return r; 127 } 128 Serialize(llvm::json::OStream & s) const129 void StructuredPythonObject::Serialize(llvm::json::OStream &s) const { 130 s.value(llvm::formatv("Python Obj: {0:X}", GetValue()).str()); 131 } 132 133 // PythonObject 134 Dump(Stream & strm) const135 void PythonObject::Dump(Stream &strm) const { 136 if (m_py_obj) { 137 FILE *file = llvm::sys::RetryAfterSignal(nullptr, ::tmpfile); 138 if (file) { 139 ::PyObject_Print(m_py_obj, file, 0); 140 const long length = ftell(file); 141 if (length) { 142 ::rewind(file); 143 std::vector<char> file_contents(length, '\0'); 144 const size_t length_read = 145 ::fread(file_contents.data(), 1, file_contents.size(), file); 146 if (length_read > 0) 147 strm.Write(file_contents.data(), length_read); 148 } 149 ::fclose(file); 150 } 151 } else 152 strm.PutCString("NULL"); 153 } 154 GetObjectType() const155 PyObjectType PythonObject::GetObjectType() const { 156 if (!IsAllocated()) 157 return PyObjectType::None; 158 159 if (PythonModule::Check(m_py_obj)) 160 return PyObjectType::Module; 161 if (PythonList::Check(m_py_obj)) 162 return PyObjectType::List; 163 if (PythonTuple::Check(m_py_obj)) 164 return PyObjectType::Tuple; 165 if (PythonDictionary::Check(m_py_obj)) 166 return PyObjectType::Dictionary; 167 if (PythonString::Check(m_py_obj)) 168 return PyObjectType::String; 169 if (PythonBytes::Check(m_py_obj)) 170 return PyObjectType::Bytes; 171 if (PythonByteArray::Check(m_py_obj)) 172 return PyObjectType::ByteArray; 173 if (PythonBoolean::Check(m_py_obj)) 174 return PyObjectType::Boolean; 175 if (PythonInteger::Check(m_py_obj)) 176 return PyObjectType::Integer; 177 if (PythonFile::Check(m_py_obj)) 178 return PyObjectType::File; 179 if (PythonCallable::Check(m_py_obj)) 180 return PyObjectType::Callable; 181 return PyObjectType::Unknown; 182 } 183 Repr() const184 PythonString PythonObject::Repr() const { 185 if (!m_py_obj) 186 return PythonString(); 187 PyObject *repr = PyObject_Repr(m_py_obj); 188 if (!repr) 189 return PythonString(); 190 return PythonString(PyRefType::Owned, repr); 191 } 192 Str() const193 PythonString PythonObject::Str() const { 194 if (!m_py_obj) 195 return PythonString(); 196 PyObject *str = PyObject_Str(m_py_obj); 197 if (!str) 198 return PythonString(); 199 return PythonString(PyRefType::Owned, str); 200 } 201 202 PythonObject ResolveNameWithDictionary(llvm::StringRef name,const PythonDictionary & dict)203 PythonObject::ResolveNameWithDictionary(llvm::StringRef name, 204 const PythonDictionary &dict) { 205 size_t dot_pos = name.find('.'); 206 llvm::StringRef piece = name.substr(0, dot_pos); 207 PythonObject result = dict.GetItemForKey(PythonString(piece)); 208 if (dot_pos == llvm::StringRef::npos) { 209 // There was no dot, we're done. 210 return result; 211 } 212 213 // There was a dot. The remaining portion of the name should be looked up in 214 // the context of the object that was found in the dictionary. 215 return result.ResolveName(name.substr(dot_pos + 1)); 216 } 217 ResolveName(llvm::StringRef name) const218 PythonObject PythonObject::ResolveName(llvm::StringRef name) const { 219 // Resolve the name in the context of the specified object. If, for example, 220 // `this` refers to a PyModule, then this will look for `name` in this 221 // module. If `this` refers to a PyType, then it will resolve `name` as an 222 // attribute of that type. If `this` refers to an instance of an object, 223 // then it will resolve `name` as the value of the specified field. 224 // 225 // This function handles dotted names so that, for example, if `m_py_obj` 226 // refers to the `sys` module, and `name` == "path.append", then it will find 227 // the function `sys.path.append`. 228 229 size_t dot_pos = name.find('.'); 230 if (dot_pos == llvm::StringRef::npos) { 231 // No dots in the name, we should be able to find the value immediately as 232 // an attribute of `m_py_obj`. 233 return GetAttributeValue(name); 234 } 235 236 // Look up the first piece of the name, and resolve the rest as a child of 237 // that. 238 PythonObject parent = ResolveName(name.substr(0, dot_pos)); 239 if (!parent.IsAllocated()) 240 return PythonObject(); 241 242 // Tail recursion.. should be optimized by the compiler 243 return parent.ResolveName(name.substr(dot_pos + 1)); 244 } 245 HasAttribute(llvm::StringRef attr) const246 bool PythonObject::HasAttribute(llvm::StringRef attr) const { 247 if (!IsValid()) 248 return false; 249 PythonString py_attr(attr); 250 return !!PyObject_HasAttr(m_py_obj, py_attr.get()); 251 } 252 GetAttributeValue(llvm::StringRef attr) const253 PythonObject PythonObject::GetAttributeValue(llvm::StringRef attr) const { 254 if (!IsValid()) 255 return PythonObject(); 256 257 PythonString py_attr(attr); 258 if (!PyObject_HasAttr(m_py_obj, py_attr.get())) 259 return PythonObject(); 260 261 return PythonObject(PyRefType::Owned, 262 PyObject_GetAttr(m_py_obj, py_attr.get())); 263 } 264 CreateStructuredObject() const265 StructuredData::ObjectSP PythonObject::CreateStructuredObject() const { 266 assert(PyGILState_Check()); 267 switch (GetObjectType()) { 268 case PyObjectType::Dictionary: 269 return PythonDictionary(PyRefType::Borrowed, m_py_obj) 270 .CreateStructuredDictionary(); 271 case PyObjectType::Boolean: 272 return PythonBoolean(PyRefType::Borrowed, m_py_obj) 273 .CreateStructuredBoolean(); 274 case PyObjectType::Integer: { 275 StructuredData::IntegerSP int_sp = 276 PythonInteger(PyRefType::Borrowed, m_py_obj).CreateStructuredInteger(); 277 if (std::holds_alternative<StructuredData::UnsignedIntegerSP>(int_sp)) 278 return std::get<StructuredData::UnsignedIntegerSP>(int_sp); 279 if (std::holds_alternative<StructuredData::SignedIntegerSP>(int_sp)) 280 return std::get<StructuredData::SignedIntegerSP>(int_sp); 281 return nullptr; 282 }; 283 case PyObjectType::List: 284 return PythonList(PyRefType::Borrowed, m_py_obj).CreateStructuredArray(); 285 case PyObjectType::String: 286 return PythonString(PyRefType::Borrowed, m_py_obj).CreateStructuredString(); 287 case PyObjectType::Bytes: 288 return PythonBytes(PyRefType::Borrowed, m_py_obj).CreateStructuredString(); 289 case PyObjectType::ByteArray: 290 return PythonByteArray(PyRefType::Borrowed, m_py_obj) 291 .CreateStructuredString(); 292 case PyObjectType::None: 293 return StructuredData::ObjectSP(); 294 default: 295 return StructuredData::ObjectSP(new StructuredPythonObject( 296 PythonObject(PyRefType::Borrowed, m_py_obj))); 297 } 298 } 299 300 // PythonString 301 PythonBytes(llvm::ArrayRef<uint8_t> bytes)302 PythonBytes::PythonBytes(llvm::ArrayRef<uint8_t> bytes) { SetBytes(bytes); } 303 PythonBytes(const uint8_t * bytes,size_t length)304 PythonBytes::PythonBytes(const uint8_t *bytes, size_t length) { 305 SetBytes(llvm::ArrayRef<uint8_t>(bytes, length)); 306 } 307 Check(PyObject * py_obj)308 bool PythonBytes::Check(PyObject *py_obj) { 309 if (!py_obj) 310 return false; 311 return PyBytes_Check(py_obj); 312 } 313 GetBytes() const314 llvm::ArrayRef<uint8_t> PythonBytes::GetBytes() const { 315 if (!IsValid()) 316 return llvm::ArrayRef<uint8_t>(); 317 318 Py_ssize_t size; 319 char *c; 320 321 PyBytes_AsStringAndSize(m_py_obj, &c, &size); 322 return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size); 323 } 324 GetSize() const325 size_t PythonBytes::GetSize() const { 326 if (!IsValid()) 327 return 0; 328 return PyBytes_Size(m_py_obj); 329 } 330 SetBytes(llvm::ArrayRef<uint8_t> bytes)331 void PythonBytes::SetBytes(llvm::ArrayRef<uint8_t> bytes) { 332 const char *data = reinterpret_cast<const char *>(bytes.data()); 333 *this = Take<PythonBytes>(PyBytes_FromStringAndSize(data, bytes.size())); 334 } 335 CreateStructuredString() const336 StructuredData::StringSP PythonBytes::CreateStructuredString() const { 337 StructuredData::StringSP result(new StructuredData::String); 338 Py_ssize_t size; 339 char *c; 340 PyBytes_AsStringAndSize(m_py_obj, &c, &size); 341 result->SetValue(std::string(c, size)); 342 return result; 343 } 344 PythonByteArray(llvm::ArrayRef<uint8_t> bytes)345 PythonByteArray::PythonByteArray(llvm::ArrayRef<uint8_t> bytes) 346 : PythonByteArray(bytes.data(), bytes.size()) {} 347 PythonByteArray(const uint8_t * bytes,size_t length)348 PythonByteArray::PythonByteArray(const uint8_t *bytes, size_t length) { 349 const char *str = reinterpret_cast<const char *>(bytes); 350 *this = Take<PythonByteArray>(PyByteArray_FromStringAndSize(str, length)); 351 } 352 Check(PyObject * py_obj)353 bool PythonByteArray::Check(PyObject *py_obj) { 354 if (!py_obj) 355 return false; 356 return PyByteArray_Check(py_obj); 357 } 358 GetBytes() const359 llvm::ArrayRef<uint8_t> PythonByteArray::GetBytes() const { 360 if (!IsValid()) 361 return llvm::ArrayRef<uint8_t>(); 362 363 char *c = PyByteArray_AsString(m_py_obj); 364 size_t size = GetSize(); 365 return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size); 366 } 367 GetSize() const368 size_t PythonByteArray::GetSize() const { 369 if (!IsValid()) 370 return 0; 371 372 return PyByteArray_Size(m_py_obj); 373 } 374 CreateStructuredString() const375 StructuredData::StringSP PythonByteArray::CreateStructuredString() const { 376 StructuredData::StringSP result(new StructuredData::String); 377 llvm::ArrayRef<uint8_t> bytes = GetBytes(); 378 const char *str = reinterpret_cast<const char *>(bytes.data()); 379 result->SetValue(std::string(str, bytes.size())); 380 return result; 381 } 382 383 // PythonString 384 FromUTF8(llvm::StringRef string)385 Expected<PythonString> PythonString::FromUTF8(llvm::StringRef string) { 386 PyObject *str = PyUnicode_FromStringAndSize(string.data(), string.size()); 387 if (!str) 388 return llvm::make_error<PythonException>(); 389 return Take<PythonString>(str); 390 } 391 PythonString(llvm::StringRef string)392 PythonString::PythonString(llvm::StringRef string) { SetString(string); } 393 Check(PyObject * py_obj)394 bool PythonString::Check(PyObject *py_obj) { 395 if (!py_obj) 396 return false; 397 398 if (PyUnicode_Check(py_obj)) 399 return true; 400 return false; 401 } 402 GetString() const403 llvm::StringRef PythonString::GetString() const { 404 auto s = AsUTF8(); 405 if (!s) { 406 llvm::consumeError(s.takeError()); 407 return llvm::StringRef(""); 408 } 409 return s.get(); 410 } 411 AsUTF8() const412 Expected<llvm::StringRef> PythonString::AsUTF8() const { 413 if (!IsValid()) 414 return nullDeref(); 415 416 Py_ssize_t size; 417 const char *data; 418 419 data = PyUnicode_AsUTF8AndSize(m_py_obj, &size); 420 421 if (!data) 422 return exception(); 423 424 return llvm::StringRef(data, size); 425 } 426 GetSize() const427 size_t PythonString::GetSize() const { 428 if (IsValid()) { 429 #if PY_MINOR_VERSION >= 3 430 return PyUnicode_GetLength(m_py_obj); 431 #else 432 return PyUnicode_GetSize(m_py_obj); 433 #endif 434 } 435 return 0; 436 } 437 SetString(llvm::StringRef string)438 void PythonString::SetString(llvm::StringRef string) { 439 auto s = FromUTF8(string); 440 if (!s) { 441 llvm::consumeError(s.takeError()); 442 Reset(); 443 } else { 444 *this = std::move(s.get()); 445 } 446 } 447 CreateStructuredString() const448 StructuredData::StringSP PythonString::CreateStructuredString() const { 449 StructuredData::StringSP result(new StructuredData::String); 450 result->SetValue(GetString()); 451 return result; 452 } 453 454 // PythonInteger 455 PythonInteger(int64_t value)456 PythonInteger::PythonInteger(int64_t value) { SetInteger(value); } 457 Check(PyObject * py_obj)458 bool PythonInteger::Check(PyObject *py_obj) { 459 if (!py_obj) 460 return false; 461 462 // Python 3 does not have PyInt_Check. There is only one type of integral 463 // value, long. 464 return PyLong_Check(py_obj); 465 } 466 SetInteger(int64_t value)467 void PythonInteger::SetInteger(int64_t value) { 468 *this = Take<PythonInteger>(PyLong_FromLongLong(value)); 469 } 470 CreateStructuredInteger() const471 StructuredData::IntegerSP PythonInteger::CreateStructuredInteger() const { 472 StructuredData::UnsignedIntegerSP uint_sp = CreateStructuredUnsignedInteger(); 473 return uint_sp ? StructuredData::IntegerSP(uint_sp) 474 : CreateStructuredSignedInteger(); 475 } 476 477 StructuredData::UnsignedIntegerSP CreateStructuredUnsignedInteger() const478 PythonInteger::CreateStructuredUnsignedInteger() const { 479 StructuredData::UnsignedIntegerSP result = nullptr; 480 llvm::Expected<unsigned long long> value = AsUnsignedLongLong(); 481 if (!value) 482 llvm::consumeError(value.takeError()); 483 else 484 result = std::make_shared<StructuredData::UnsignedInteger>(value.get()); 485 486 return result; 487 } 488 489 StructuredData::SignedIntegerSP CreateStructuredSignedInteger() const490 PythonInteger::CreateStructuredSignedInteger() const { 491 StructuredData::SignedIntegerSP result = nullptr; 492 llvm::Expected<long long> value = AsLongLong(); 493 if (!value) 494 llvm::consumeError(value.takeError()); 495 else 496 result = std::make_shared<StructuredData::SignedInteger>(value.get()); 497 498 return result; 499 } 500 501 // PythonBoolean 502 PythonBoolean(bool value)503 PythonBoolean::PythonBoolean(bool value) { 504 SetValue(value); 505 } 506 Check(PyObject * py_obj)507 bool PythonBoolean::Check(PyObject *py_obj) { 508 return py_obj ? PyBool_Check(py_obj) : false; 509 } 510 GetValue() const511 bool PythonBoolean::GetValue() const { 512 return m_py_obj ? PyObject_IsTrue(m_py_obj) : false; 513 } 514 SetValue(bool value)515 void PythonBoolean::SetValue(bool value) { 516 *this = Take<PythonBoolean>(PyBool_FromLong(value)); 517 } 518 CreateStructuredBoolean() const519 StructuredData::BooleanSP PythonBoolean::CreateStructuredBoolean() const { 520 StructuredData::BooleanSP result(new StructuredData::Boolean); 521 result->SetValue(GetValue()); 522 return result; 523 } 524 525 // PythonList 526 PythonList(PyInitialValue value)527 PythonList::PythonList(PyInitialValue value) { 528 if (value == PyInitialValue::Empty) 529 *this = Take<PythonList>(PyList_New(0)); 530 } 531 PythonList(int list_size)532 PythonList::PythonList(int list_size) { 533 *this = Take<PythonList>(PyList_New(list_size)); 534 } 535 Check(PyObject * py_obj)536 bool PythonList::Check(PyObject *py_obj) { 537 if (!py_obj) 538 return false; 539 return PyList_Check(py_obj); 540 } 541 GetSize() const542 uint32_t PythonList::GetSize() const { 543 if (IsValid()) 544 return PyList_GET_SIZE(m_py_obj); 545 return 0; 546 } 547 GetItemAtIndex(uint32_t index) const548 PythonObject PythonList::GetItemAtIndex(uint32_t index) const { 549 if (IsValid()) 550 return PythonObject(PyRefType::Borrowed, PyList_GetItem(m_py_obj, index)); 551 return PythonObject(); 552 } 553 SetItemAtIndex(uint32_t index,const PythonObject & object)554 void PythonList::SetItemAtIndex(uint32_t index, const PythonObject &object) { 555 if (IsAllocated() && object.IsValid()) { 556 // PyList_SetItem is documented to "steal" a reference, so we need to 557 // convert it to an owned reference by incrementing it. 558 Py_INCREF(object.get()); 559 PyList_SetItem(m_py_obj, index, object.get()); 560 } 561 } 562 AppendItem(const PythonObject & object)563 void PythonList::AppendItem(const PythonObject &object) { 564 if (IsAllocated() && object.IsValid()) { 565 // `PyList_Append` does *not* steal a reference, so do not call `Py_INCREF` 566 // here like we do with `PyList_SetItem`. 567 PyList_Append(m_py_obj, object.get()); 568 } 569 } 570 CreateStructuredArray() const571 StructuredData::ArraySP PythonList::CreateStructuredArray() const { 572 StructuredData::ArraySP result(new StructuredData::Array); 573 uint32_t count = GetSize(); 574 for (uint32_t i = 0; i < count; ++i) { 575 PythonObject obj = GetItemAtIndex(i); 576 result->AddItem(obj.CreateStructuredObject()); 577 } 578 return result; 579 } 580 581 // PythonTuple 582 PythonTuple(PyInitialValue value)583 PythonTuple::PythonTuple(PyInitialValue value) { 584 if (value == PyInitialValue::Empty) 585 *this = Take<PythonTuple>(PyTuple_New(0)); 586 } 587 PythonTuple(int tuple_size)588 PythonTuple::PythonTuple(int tuple_size) { 589 *this = Take<PythonTuple>(PyTuple_New(tuple_size)); 590 } 591 PythonTuple(std::initializer_list<PythonObject> objects)592 PythonTuple::PythonTuple(std::initializer_list<PythonObject> objects) { 593 m_py_obj = PyTuple_New(objects.size()); 594 595 uint32_t idx = 0; 596 for (auto object : objects) { 597 if (object.IsValid()) 598 SetItemAtIndex(idx, object); 599 idx++; 600 } 601 } 602 PythonTuple(std::initializer_list<PyObject * > objects)603 PythonTuple::PythonTuple(std::initializer_list<PyObject *> objects) { 604 m_py_obj = PyTuple_New(objects.size()); 605 606 uint32_t idx = 0; 607 for (auto py_object : objects) { 608 PythonObject object(PyRefType::Borrowed, py_object); 609 if (object.IsValid()) 610 SetItemAtIndex(idx, object); 611 idx++; 612 } 613 } 614 Check(PyObject * py_obj)615 bool PythonTuple::Check(PyObject *py_obj) { 616 if (!py_obj) 617 return false; 618 return PyTuple_Check(py_obj); 619 } 620 GetSize() const621 uint32_t PythonTuple::GetSize() const { 622 if (IsValid()) 623 return PyTuple_GET_SIZE(m_py_obj); 624 return 0; 625 } 626 GetItemAtIndex(uint32_t index) const627 PythonObject PythonTuple::GetItemAtIndex(uint32_t index) const { 628 if (IsValid()) 629 return PythonObject(PyRefType::Borrowed, PyTuple_GetItem(m_py_obj, index)); 630 return PythonObject(); 631 } 632 SetItemAtIndex(uint32_t index,const PythonObject & object)633 void PythonTuple::SetItemAtIndex(uint32_t index, const PythonObject &object) { 634 if (IsAllocated() && object.IsValid()) { 635 // PyTuple_SetItem is documented to "steal" a reference, so we need to 636 // convert it to an owned reference by incrementing it. 637 Py_INCREF(object.get()); 638 PyTuple_SetItem(m_py_obj, index, object.get()); 639 } 640 } 641 CreateStructuredArray() const642 StructuredData::ArraySP PythonTuple::CreateStructuredArray() const { 643 StructuredData::ArraySP result(new StructuredData::Array); 644 uint32_t count = GetSize(); 645 for (uint32_t i = 0; i < count; ++i) { 646 PythonObject obj = GetItemAtIndex(i); 647 result->AddItem(obj.CreateStructuredObject()); 648 } 649 return result; 650 } 651 652 // PythonDictionary 653 PythonDictionary(PyInitialValue value)654 PythonDictionary::PythonDictionary(PyInitialValue value) { 655 if (value == PyInitialValue::Empty) 656 *this = Take<PythonDictionary>(PyDict_New()); 657 } 658 Check(PyObject * py_obj)659 bool PythonDictionary::Check(PyObject *py_obj) { 660 if (!py_obj) 661 return false; 662 663 return PyDict_Check(py_obj); 664 } 665 HasKey(const llvm::Twine & key) const666 bool PythonDictionary::HasKey(const llvm::Twine &key) const { 667 if (!IsValid()) 668 return false; 669 670 PythonString key_object(key.isSingleStringRef() ? key.getSingleStringRef() 671 : key.str()); 672 673 if (int res = PyDict_Contains(m_py_obj, key_object.get()) > 0) 674 return res; 675 676 PyErr_Print(); 677 return false; 678 } 679 GetSize() const680 uint32_t PythonDictionary::GetSize() const { 681 if (IsValid()) 682 return PyDict_Size(m_py_obj); 683 return 0; 684 } 685 GetKeys() const686 PythonList PythonDictionary::GetKeys() const { 687 if (IsValid()) 688 return PythonList(PyRefType::Owned, PyDict_Keys(m_py_obj)); 689 return PythonList(PyInitialValue::Invalid); 690 } 691 GetItemForKey(const PythonObject & key) const692 PythonObject PythonDictionary::GetItemForKey(const PythonObject &key) const { 693 auto item = GetItem(key); 694 if (!item) { 695 llvm::consumeError(item.takeError()); 696 return PythonObject(); 697 } 698 return std::move(item.get()); 699 } 700 701 Expected<PythonObject> GetItem(const PythonObject & key) const702 PythonDictionary::GetItem(const PythonObject &key) const { 703 if (!IsValid()) 704 return nullDeref(); 705 PyObject *o = PyDict_GetItemWithError(m_py_obj, key.get()); 706 if (PyErr_Occurred()) 707 return exception(); 708 if (!o) 709 return keyError(); 710 return Retain<PythonObject>(o); 711 } 712 GetItem(const Twine & key) const713 Expected<PythonObject> PythonDictionary::GetItem(const Twine &key) const { 714 if (!IsValid()) 715 return nullDeref(); 716 PyObject *o = PyDict_GetItemString(m_py_obj, NullTerminated(key)); 717 if (PyErr_Occurred()) 718 return exception(); 719 if (!o) 720 return keyError(); 721 return Retain<PythonObject>(o); 722 } 723 SetItem(const PythonObject & key,const PythonObject & value) const724 Error PythonDictionary::SetItem(const PythonObject &key, 725 const PythonObject &value) const { 726 if (!IsValid() || !value.IsValid()) 727 return nullDeref(); 728 int r = PyDict_SetItem(m_py_obj, key.get(), value.get()); 729 if (r < 0) 730 return exception(); 731 return Error::success(); 732 } 733 SetItem(const Twine & key,const PythonObject & value) const734 Error PythonDictionary::SetItem(const Twine &key, 735 const PythonObject &value) const { 736 if (!IsValid() || !value.IsValid()) 737 return nullDeref(); 738 int r = PyDict_SetItemString(m_py_obj, NullTerminated(key), value.get()); 739 if (r < 0) 740 return exception(); 741 return Error::success(); 742 } 743 SetItemForKey(const PythonObject & key,const PythonObject & value)744 void PythonDictionary::SetItemForKey(const PythonObject &key, 745 const PythonObject &value) { 746 Error error = SetItem(key, value); 747 if (error) 748 llvm::consumeError(std::move(error)); 749 } 750 751 StructuredData::DictionarySP CreateStructuredDictionary() const752 PythonDictionary::CreateStructuredDictionary() const { 753 StructuredData::DictionarySP result(new StructuredData::Dictionary); 754 PythonList keys(GetKeys()); 755 uint32_t num_keys = keys.GetSize(); 756 for (uint32_t i = 0; i < num_keys; ++i) { 757 PythonObject key = keys.GetItemAtIndex(i); 758 PythonObject value = GetItemForKey(key); 759 StructuredData::ObjectSP structured_value = value.CreateStructuredObject(); 760 result->AddItem(key.Str().GetString(), structured_value); 761 } 762 return result; 763 } 764 BuiltinsModule()765 PythonModule PythonModule::BuiltinsModule() { return AddModule("builtins"); } 766 MainModule()767 PythonModule PythonModule::MainModule() { return AddModule("__main__"); } 768 AddModule(llvm::StringRef module)769 PythonModule PythonModule::AddModule(llvm::StringRef module) { 770 std::string str = module.str(); 771 return PythonModule(PyRefType::Borrowed, PyImport_AddModule(str.c_str())); 772 } 773 Import(const Twine & name)774 Expected<PythonModule> PythonModule::Import(const Twine &name) { 775 PyObject *mod = PyImport_ImportModule(NullTerminated(name)); 776 if (!mod) 777 return exception(); 778 return Take<PythonModule>(mod); 779 } 780 Get(const Twine & name)781 Expected<PythonObject> PythonModule::Get(const Twine &name) { 782 if (!IsValid()) 783 return nullDeref(); 784 PyObject *dict = PyModule_GetDict(m_py_obj); 785 if (!dict) 786 return exception(); 787 PyObject *item = PyDict_GetItemString(dict, NullTerminated(name)); 788 if (!item) 789 return exception(); 790 return Retain<PythonObject>(item); 791 } 792 Check(PyObject * py_obj)793 bool PythonModule::Check(PyObject *py_obj) { 794 if (!py_obj) 795 return false; 796 797 return PyModule_Check(py_obj); 798 } 799 GetDictionary() const800 PythonDictionary PythonModule::GetDictionary() const { 801 if (!IsValid()) 802 return PythonDictionary(); 803 return Retain<PythonDictionary>(PyModule_GetDict(m_py_obj)); 804 } 805 Check(PyObject * py_obj)806 bool PythonCallable::Check(PyObject *py_obj) { 807 if (!py_obj) 808 return false; 809 810 return PyCallable_Check(py_obj); 811 } 812 813 #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 814 static const char get_arg_info_script[] = R"( 815 from inspect import signature, Parameter, ismethod 816 from collections import namedtuple 817 ArgInfo = namedtuple('ArgInfo', ['count', 'has_varargs']) 818 def main(f): 819 count = 0 820 varargs = False 821 for parameter in signature(f).parameters.values(): 822 kind = parameter.kind 823 if kind in (Parameter.POSITIONAL_ONLY, 824 Parameter.POSITIONAL_OR_KEYWORD): 825 count += 1 826 elif kind == Parameter.VAR_POSITIONAL: 827 varargs = True 828 elif kind in (Parameter.KEYWORD_ONLY, 829 Parameter.VAR_KEYWORD): 830 pass 831 else: 832 raise Exception(f'unknown parameter kind: {kind}') 833 return ArgInfo(count, varargs) 834 )"; 835 #endif 836 GetArgInfo() const837 Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const { 838 ArgInfo result = {}; 839 if (!IsValid()) 840 return nullDeref(); 841 842 #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 843 844 // no need to synchronize access to this global, we already have the GIL 845 static PythonScript get_arg_info(get_arg_info_script); 846 Expected<PythonObject> pyarginfo = get_arg_info(*this); 847 if (!pyarginfo) 848 return pyarginfo.takeError(); 849 long long count = 850 cantFail(As<long long>(pyarginfo.get().GetAttribute("count"))); 851 bool has_varargs = 852 cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs"))); 853 result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count; 854 855 #else 856 PyObject *py_func_obj; 857 bool is_bound_method = false; 858 bool is_class = false; 859 860 if (PyType_Check(m_py_obj) || PyClass_Check(m_py_obj)) { 861 auto init = GetAttribute("__init__"); 862 if (!init) 863 return init.takeError(); 864 py_func_obj = init.get().get(); 865 is_class = true; 866 } else { 867 py_func_obj = m_py_obj; 868 } 869 870 if (PyMethod_Check(py_func_obj)) { 871 py_func_obj = PyMethod_GET_FUNCTION(py_func_obj); 872 PythonObject im_self = GetAttributeValue("im_self"); 873 if (im_self.IsValid() && !im_self.IsNone()) 874 is_bound_method = true; 875 } else { 876 // see if this is a callable object with an __call__ method 877 if (!PyFunction_Check(py_func_obj)) { 878 PythonObject __call__ = GetAttributeValue("__call__"); 879 if (__call__.IsValid()) { 880 auto __callable__ = __call__.AsType<PythonCallable>(); 881 if (__callable__.IsValid()) { 882 py_func_obj = PyMethod_GET_FUNCTION(__callable__.get()); 883 PythonObject im_self = __callable__.GetAttributeValue("im_self"); 884 if (im_self.IsValid() && !im_self.IsNone()) 885 is_bound_method = true; 886 } 887 } 888 } 889 } 890 891 if (!py_func_obj) 892 return result; 893 894 PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(py_func_obj); 895 if (!code) 896 return result; 897 898 auto count = code->co_argcount; 899 bool has_varargs = !!(code->co_flags & CO_VARARGS); 900 result.max_positional_args = 901 has_varargs ? ArgInfo::UNBOUNDED 902 : (count - (int)is_bound_method) - (int)is_class; 903 904 #endif 905 906 return result; 907 } 908 909 constexpr unsigned 910 PythonCallable::ArgInfo::UNBOUNDED; // FIXME delete after c++17 911 operator ()()912 PythonObject PythonCallable::operator()() { 913 return PythonObject(PyRefType::Owned, PyObject_CallObject(m_py_obj, nullptr)); 914 } 915 916 PythonObject PythonCallable:: operator ()(std::initializer_list<PyObject * > args)917 operator()(std::initializer_list<PyObject *> args) { 918 PythonTuple arg_tuple(args); 919 return PythonObject(PyRefType::Owned, 920 PyObject_CallObject(m_py_obj, arg_tuple.get())); 921 } 922 923 PythonObject PythonCallable:: operator ()(std::initializer_list<PythonObject> args)924 operator()(std::initializer_list<PythonObject> args) { 925 PythonTuple arg_tuple(args); 926 return PythonObject(PyRefType::Owned, 927 PyObject_CallObject(m_py_obj, arg_tuple.get())); 928 } 929 Check(PyObject * py_obj)930 bool PythonFile::Check(PyObject *py_obj) { 931 if (!py_obj) 932 return false; 933 // In Python 3, there is no `PyFile_Check`, and in fact PyFile is not even a 934 // first-class object type anymore. `PyFile_FromFd` is just a thin wrapper 935 // over `io.open()`, which returns some object derived from `io.IOBase`. As a 936 // result, the only way to detect a file in Python 3 is to check whether it 937 // inherits from `io.IOBase`. 938 auto io_module = PythonModule::Import("io"); 939 if (!io_module) { 940 llvm::consumeError(io_module.takeError()); 941 return false; 942 } 943 auto iobase = io_module.get().Get("IOBase"); 944 if (!iobase) { 945 llvm::consumeError(iobase.takeError()); 946 return false; 947 } 948 int r = PyObject_IsInstance(py_obj, iobase.get().get()); 949 if (r < 0) { 950 llvm::consumeError(exception()); // clear the exception and log it. 951 return false; 952 } 953 return !!r; 954 } 955 toCString() const956 const char *PythonException::toCString() const { 957 if (!m_repr_bytes) 958 return "unknown exception"; 959 return PyBytes_AS_STRING(m_repr_bytes); 960 } 961 PythonException(const char * caller)962 PythonException::PythonException(const char *caller) { 963 assert(PyErr_Occurred()); 964 m_exception_type = m_exception = m_traceback = m_repr_bytes = nullptr; 965 PyErr_Fetch(&m_exception_type, &m_exception, &m_traceback); 966 PyErr_NormalizeException(&m_exception_type, &m_exception, &m_traceback); 967 PyErr_Clear(); 968 if (m_exception) { 969 PyObject *repr = PyObject_Repr(m_exception); 970 if (repr) { 971 m_repr_bytes = PyUnicode_AsEncodedString(repr, "utf-8", nullptr); 972 if (!m_repr_bytes) { 973 PyErr_Clear(); 974 } 975 Py_XDECREF(repr); 976 } else { 977 PyErr_Clear(); 978 } 979 } 980 Log *log = GetLog(LLDBLog::Script); 981 if (caller) 982 LLDB_LOGF(log, "%s failed with exception: %s", caller, toCString()); 983 else 984 LLDB_LOGF(log, "python exception: %s", toCString()); 985 } Restore()986 void PythonException::Restore() { 987 if (m_exception_type && m_exception) { 988 PyErr_Restore(m_exception_type, m_exception, m_traceback); 989 } else { 990 PyErr_SetString(PyExc_Exception, toCString()); 991 } 992 m_exception_type = m_exception = m_traceback = nullptr; 993 } 994 ~PythonException()995 PythonException::~PythonException() { 996 Py_XDECREF(m_exception_type); 997 Py_XDECREF(m_exception); 998 Py_XDECREF(m_traceback); 999 Py_XDECREF(m_repr_bytes); 1000 } 1001 log(llvm::raw_ostream & OS) const1002 void PythonException::log(llvm::raw_ostream &OS) const { OS << toCString(); } 1003 convertToErrorCode() const1004 std::error_code PythonException::convertToErrorCode() const { 1005 return llvm::inconvertibleErrorCode(); 1006 } 1007 Matches(PyObject * exc) const1008 bool PythonException::Matches(PyObject *exc) const { 1009 return PyErr_GivenExceptionMatches(m_exception_type, exc); 1010 } 1011 1012 const char read_exception_script[] = R"( 1013 import sys 1014 from traceback import print_exception 1015 if sys.version_info.major < 3: 1016 from StringIO import StringIO 1017 else: 1018 from io import StringIO 1019 def main(exc_type, exc_value, tb): 1020 f = StringIO() 1021 print_exception(exc_type, exc_value, tb, file=f) 1022 return f.getvalue() 1023 )"; 1024 ReadBacktrace() const1025 std::string PythonException::ReadBacktrace() const { 1026 1027 if (!m_traceback) 1028 return toCString(); 1029 1030 // no need to synchronize access to this global, we already have the GIL 1031 static PythonScript read_exception(read_exception_script); 1032 1033 Expected<std::string> backtrace = As<std::string>( 1034 read_exception(m_exception_type, m_exception, m_traceback)); 1035 1036 if (!backtrace) { 1037 std::string message = 1038 std::string(toCString()) + "\n" + 1039 "Traceback unavailable, an error occurred while reading it:\n"; 1040 return (message + llvm::toString(backtrace.takeError())); 1041 } 1042 1043 return std::move(backtrace.get()); 1044 } 1045 1046 char PythonException::ID = 0; 1047 1048 llvm::Expected<File::OpenOptions> GetOptionsForPyObject(const PythonObject & obj)1049 GetOptionsForPyObject(const PythonObject &obj) { 1050 auto options = File::OpenOptions(0); 1051 auto readable = As<bool>(obj.CallMethod("readable")); 1052 if (!readable) 1053 return readable.takeError(); 1054 auto writable = As<bool>(obj.CallMethod("writable")); 1055 if (!writable) 1056 return writable.takeError(); 1057 if (readable.get() && writable.get()) 1058 options |= File::eOpenOptionReadWrite; 1059 else if (writable.get()) 1060 options |= File::eOpenOptionWriteOnly; 1061 else if (readable.get()) 1062 options |= File::eOpenOptionReadOnly; 1063 return options; 1064 } 1065 1066 // Base class template for python files. All it knows how to do 1067 // is hold a reference to the python object and close or flush it 1068 // when the File is closed. 1069 namespace { 1070 template <typename Base> class OwnedPythonFile : public Base { 1071 public: 1072 template <typename... Args> OwnedPythonFile(const PythonFile & file,bool borrowed,Args...args)1073 OwnedPythonFile(const PythonFile &file, bool borrowed, Args... args) 1074 : Base(args...), m_py_obj(file), m_borrowed(borrowed) { 1075 assert(m_py_obj); 1076 } 1077 ~OwnedPythonFile()1078 ~OwnedPythonFile() override { 1079 assert(m_py_obj); 1080 GIL takeGIL; 1081 Close(); 1082 // we need to ensure the python object is released while we still 1083 // hold the GIL 1084 m_py_obj.Reset(); 1085 } 1086 IsPythonSideValid() const1087 bool IsPythonSideValid() const { 1088 GIL takeGIL; 1089 auto closed = As<bool>(m_py_obj.GetAttribute("closed")); 1090 if (!closed) { 1091 llvm::consumeError(closed.takeError()); 1092 return false; 1093 } 1094 return !closed.get(); 1095 } 1096 IsValid() const1097 bool IsValid() const override { 1098 return IsPythonSideValid() && Base::IsValid(); 1099 } 1100 Close()1101 Status Close() override { 1102 assert(m_py_obj); 1103 Status py_error, base_error; 1104 GIL takeGIL; 1105 if (!m_borrowed) { 1106 auto r = m_py_obj.CallMethod("close"); 1107 if (!r) 1108 py_error = Status(r.takeError()); 1109 } 1110 base_error = Base::Close(); 1111 if (py_error.Fail()) 1112 return py_error; 1113 return base_error; 1114 }; 1115 GetPythonObject() const1116 PyObject *GetPythonObject() const { 1117 assert(m_py_obj.IsValid()); 1118 return m_py_obj.get(); 1119 } 1120 1121 static bool classof(const File *file) = delete; 1122 1123 protected: 1124 PythonFile m_py_obj; 1125 bool m_borrowed; 1126 }; 1127 } // namespace 1128 1129 // A SimplePythonFile is a OwnedPythonFile that just does all I/O as 1130 // a NativeFile 1131 namespace { 1132 class SimplePythonFile : public OwnedPythonFile<NativeFile> { 1133 public: SimplePythonFile(const PythonFile & file,bool borrowed,int fd,File::OpenOptions options)1134 SimplePythonFile(const PythonFile &file, bool borrowed, int fd, 1135 File::OpenOptions options) 1136 : OwnedPythonFile(file, borrowed, fd, options, false) {} 1137 1138 static char ID; isA(const void * classID) const1139 bool isA(const void *classID) const override { 1140 return classID == &ID || NativeFile::isA(classID); 1141 } classof(const File * file)1142 static bool classof(const File *file) { return file->isA(&ID); } 1143 }; 1144 char SimplePythonFile::ID = 0; 1145 } // namespace 1146 1147 namespace { 1148 class PythonBuffer { 1149 public: 1150 PythonBuffer &operator=(const PythonBuffer &) = delete; 1151 PythonBuffer(const PythonBuffer &) = delete; 1152 Create(PythonObject & obj,int flags=PyBUF_SIMPLE)1153 static Expected<PythonBuffer> Create(PythonObject &obj, 1154 int flags = PyBUF_SIMPLE) { 1155 Py_buffer py_buffer = {}; 1156 PyObject_GetBuffer(obj.get(), &py_buffer, flags); 1157 if (!py_buffer.obj) 1158 return llvm::make_error<PythonException>(); 1159 return PythonBuffer(py_buffer); 1160 } 1161 PythonBuffer(PythonBuffer && other)1162 PythonBuffer(PythonBuffer &&other) { 1163 m_buffer = other.m_buffer; 1164 other.m_buffer.obj = nullptr; 1165 } 1166 ~PythonBuffer()1167 ~PythonBuffer() { 1168 if (m_buffer.obj) 1169 PyBuffer_Release(&m_buffer); 1170 } 1171 get()1172 Py_buffer &get() { return m_buffer; } 1173 1174 private: 1175 // takes ownership of the buffer. PythonBuffer(const Py_buffer & py_buffer)1176 PythonBuffer(const Py_buffer &py_buffer) : m_buffer(py_buffer) {} 1177 Py_buffer m_buffer; 1178 }; 1179 } // namespace 1180 1181 // Shared methods between TextPythonFile and BinaryPythonFile 1182 namespace { 1183 class PythonIOFile : public OwnedPythonFile<File> { 1184 public: PythonIOFile(const PythonFile & file,bool borrowed)1185 PythonIOFile(const PythonFile &file, bool borrowed) 1186 : OwnedPythonFile(file, borrowed) {} 1187 ~PythonIOFile()1188 ~PythonIOFile() override { Close(); } 1189 IsValid() const1190 bool IsValid() const override { return IsPythonSideValid(); } 1191 Close()1192 Status Close() override { 1193 assert(m_py_obj); 1194 GIL takeGIL; 1195 if (m_borrowed) 1196 return Flush(); 1197 auto r = m_py_obj.CallMethod("close"); 1198 if (!r) 1199 return Status(r.takeError()); 1200 return Status(); 1201 } 1202 Flush()1203 Status Flush() override { 1204 GIL takeGIL; 1205 auto r = m_py_obj.CallMethod("flush"); 1206 if (!r) 1207 return Status(r.takeError()); 1208 return Status(); 1209 } 1210 GetOptions() const1211 Expected<File::OpenOptions> GetOptions() const override { 1212 GIL takeGIL; 1213 return GetOptionsForPyObject(m_py_obj); 1214 } 1215 1216 static char ID; isA(const void * classID) const1217 bool isA(const void *classID) const override { 1218 return classID == &ID || File::isA(classID); 1219 } classof(const File * file)1220 static bool classof(const File *file) { return file->isA(&ID); } 1221 }; 1222 char PythonIOFile::ID = 0; 1223 } // namespace 1224 1225 namespace { 1226 class BinaryPythonFile : public PythonIOFile { 1227 protected: 1228 int m_descriptor; 1229 1230 public: BinaryPythonFile(int fd,const PythonFile & file,bool borrowed)1231 BinaryPythonFile(int fd, const PythonFile &file, bool borrowed) 1232 : PythonIOFile(file, borrowed), 1233 m_descriptor(File::DescriptorIsValid(fd) ? fd 1234 : File::kInvalidDescriptor) {} 1235 GetDescriptor() const1236 int GetDescriptor() const override { return m_descriptor; } 1237 Write(const void * buf,size_t & num_bytes)1238 Status Write(const void *buf, size_t &num_bytes) override { 1239 GIL takeGIL; 1240 PyObject *pybuffer_p = PyMemoryView_FromMemory( 1241 const_cast<char *>((const char *)buf), num_bytes, PyBUF_READ); 1242 if (!pybuffer_p) 1243 return Status(llvm::make_error<PythonException>()); 1244 auto pybuffer = Take<PythonObject>(pybuffer_p); 1245 num_bytes = 0; 1246 auto bytes_written = As<long long>(m_py_obj.CallMethod("write", pybuffer)); 1247 if (!bytes_written) 1248 return Status(bytes_written.takeError()); 1249 if (bytes_written.get() < 0) 1250 return Status(".write() method returned a negative number!"); 1251 static_assert(sizeof(long long) >= sizeof(size_t), "overflow"); 1252 num_bytes = bytes_written.get(); 1253 return Status(); 1254 } 1255 Read(void * buf,size_t & num_bytes)1256 Status Read(void *buf, size_t &num_bytes) override { 1257 GIL takeGIL; 1258 static_assert(sizeof(long long) >= sizeof(size_t), "overflow"); 1259 auto pybuffer_obj = 1260 m_py_obj.CallMethod("read", (unsigned long long)num_bytes); 1261 if (!pybuffer_obj) 1262 return Status(pybuffer_obj.takeError()); 1263 num_bytes = 0; 1264 if (pybuffer_obj.get().IsNone()) { 1265 // EOF 1266 num_bytes = 0; 1267 return Status(); 1268 } 1269 auto pybuffer = PythonBuffer::Create(pybuffer_obj.get()); 1270 if (!pybuffer) 1271 return Status(pybuffer.takeError()); 1272 memcpy(buf, pybuffer.get().get().buf, pybuffer.get().get().len); 1273 num_bytes = pybuffer.get().get().len; 1274 return Status(); 1275 } 1276 }; 1277 } // namespace 1278 1279 namespace { 1280 class TextPythonFile : public PythonIOFile { 1281 protected: 1282 int m_descriptor; 1283 1284 public: TextPythonFile(int fd,const PythonFile & file,bool borrowed)1285 TextPythonFile(int fd, const PythonFile &file, bool borrowed) 1286 : PythonIOFile(file, borrowed), 1287 m_descriptor(File::DescriptorIsValid(fd) ? fd 1288 : File::kInvalidDescriptor) {} 1289 GetDescriptor() const1290 int GetDescriptor() const override { return m_descriptor; } 1291 Write(const void * buf,size_t & num_bytes)1292 Status Write(const void *buf, size_t &num_bytes) override { 1293 GIL takeGIL; 1294 auto pystring = 1295 PythonString::FromUTF8(llvm::StringRef((const char *)buf, num_bytes)); 1296 if (!pystring) 1297 return Status(pystring.takeError()); 1298 num_bytes = 0; 1299 auto bytes_written = 1300 As<long long>(m_py_obj.CallMethod("write", pystring.get())); 1301 if (!bytes_written) 1302 return Status(bytes_written.takeError()); 1303 if (bytes_written.get() < 0) 1304 return Status(".write() method returned a negative number!"); 1305 static_assert(sizeof(long long) >= sizeof(size_t), "overflow"); 1306 num_bytes = bytes_written.get(); 1307 return Status(); 1308 } 1309 Read(void * buf,size_t & num_bytes)1310 Status Read(void *buf, size_t &num_bytes) override { 1311 GIL takeGIL; 1312 size_t num_chars = num_bytes / 6; 1313 size_t orig_num_bytes = num_bytes; 1314 num_bytes = 0; 1315 if (orig_num_bytes < 6) { 1316 return Status("can't read less than 6 bytes from a utf8 text stream"); 1317 } 1318 auto pystring = As<PythonString>( 1319 m_py_obj.CallMethod("read", (unsigned long long)num_chars)); 1320 if (!pystring) 1321 return Status(pystring.takeError()); 1322 if (pystring.get().IsNone()) { 1323 // EOF 1324 return Status(); 1325 } 1326 auto stringref = pystring.get().AsUTF8(); 1327 if (!stringref) 1328 return Status(stringref.takeError()); 1329 num_bytes = stringref.get().size(); 1330 memcpy(buf, stringref.get().begin(), num_bytes); 1331 return Status(); 1332 } 1333 }; 1334 } // namespace 1335 ConvertToFile(bool borrowed)1336 llvm::Expected<FileSP> PythonFile::ConvertToFile(bool borrowed) { 1337 if (!IsValid()) 1338 return llvm::createStringError(llvm::inconvertibleErrorCode(), 1339 "invalid PythonFile"); 1340 1341 int fd = PyObject_AsFileDescriptor(m_py_obj); 1342 if (fd < 0) { 1343 PyErr_Clear(); 1344 return ConvertToFileForcingUseOfScriptingIOMethods(borrowed); 1345 } 1346 auto options = GetOptionsForPyObject(*this); 1347 if (!options) 1348 return options.takeError(); 1349 1350 File::OpenOptions rw = 1351 options.get() & (File::eOpenOptionReadOnly | File::eOpenOptionWriteOnly | 1352 File::eOpenOptionReadWrite); 1353 if (rw == File::eOpenOptionWriteOnly || rw == File::eOpenOptionReadWrite) { 1354 // LLDB and python will not share I/O buffers. We should probably 1355 // flush the python buffers now. 1356 auto r = CallMethod("flush"); 1357 if (!r) 1358 return r.takeError(); 1359 } 1360 1361 FileSP file_sp; 1362 if (borrowed) { 1363 // In this case we don't need to retain the python 1364 // object at all. 1365 file_sp = std::make_shared<NativeFile>(fd, options.get(), false); 1366 } else { 1367 file_sp = std::static_pointer_cast<File>( 1368 std::make_shared<SimplePythonFile>(*this, borrowed, fd, options.get())); 1369 } 1370 if (!file_sp->IsValid()) 1371 return llvm::createStringError(llvm::inconvertibleErrorCode(), 1372 "invalid File"); 1373 1374 return file_sp; 1375 } 1376 1377 llvm::Expected<FileSP> ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed)1378 PythonFile::ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed) { 1379 1380 assert(!PyErr_Occurred()); 1381 1382 if (!IsValid()) 1383 return llvm::createStringError(llvm::inconvertibleErrorCode(), 1384 "invalid PythonFile"); 1385 1386 int fd = PyObject_AsFileDescriptor(m_py_obj); 1387 if (fd < 0) { 1388 PyErr_Clear(); 1389 fd = File::kInvalidDescriptor; 1390 } 1391 1392 auto io_module = PythonModule::Import("io"); 1393 if (!io_module) 1394 return io_module.takeError(); 1395 auto textIOBase = io_module.get().Get("TextIOBase"); 1396 if (!textIOBase) 1397 return textIOBase.takeError(); 1398 auto rawIOBase = io_module.get().Get("RawIOBase"); 1399 if (!rawIOBase) 1400 return rawIOBase.takeError(); 1401 auto bufferedIOBase = io_module.get().Get("BufferedIOBase"); 1402 if (!bufferedIOBase) 1403 return bufferedIOBase.takeError(); 1404 1405 FileSP file_sp; 1406 1407 auto isTextIO = IsInstance(textIOBase.get()); 1408 if (!isTextIO) 1409 return isTextIO.takeError(); 1410 if (isTextIO.get()) 1411 file_sp = std::static_pointer_cast<File>( 1412 std::make_shared<TextPythonFile>(fd, *this, borrowed)); 1413 1414 auto isRawIO = IsInstance(rawIOBase.get()); 1415 if (!isRawIO) 1416 return isRawIO.takeError(); 1417 auto isBufferedIO = IsInstance(bufferedIOBase.get()); 1418 if (!isBufferedIO) 1419 return isBufferedIO.takeError(); 1420 1421 if (isRawIO.get() || isBufferedIO.get()) { 1422 file_sp = std::static_pointer_cast<File>( 1423 std::make_shared<BinaryPythonFile>(fd, *this, borrowed)); 1424 } 1425 1426 if (!file_sp) 1427 return llvm::createStringError(llvm::inconvertibleErrorCode(), 1428 "python file is neither text nor binary"); 1429 1430 if (!file_sp->IsValid()) 1431 return llvm::createStringError(llvm::inconvertibleErrorCode(), 1432 "invalid File"); 1433 1434 return file_sp; 1435 } 1436 FromFile(File & file,const char * mode)1437 Expected<PythonFile> PythonFile::FromFile(File &file, const char *mode) { 1438 if (!file.IsValid()) 1439 return llvm::createStringError(llvm::inconvertibleErrorCode(), 1440 "invalid file"); 1441 1442 if (auto *simple = llvm::dyn_cast<SimplePythonFile>(&file)) 1443 return Retain<PythonFile>(simple->GetPythonObject()); 1444 if (auto *pythonio = llvm::dyn_cast<PythonIOFile>(&file)) 1445 return Retain<PythonFile>(pythonio->GetPythonObject()); 1446 1447 if (!mode) { 1448 auto m = file.GetOpenMode(); 1449 if (!m) 1450 return m.takeError(); 1451 mode = m.get(); 1452 } 1453 1454 PyObject *file_obj; 1455 file_obj = PyFile_FromFd(file.GetDescriptor(), nullptr, mode, -1, nullptr, 1456 "ignore", nullptr, /*closefd=*/0); 1457 1458 if (!file_obj) 1459 return exception(); 1460 1461 return Take<PythonFile>(file_obj); 1462 } 1463 Init()1464 Error PythonScript::Init() { 1465 if (function.IsValid()) 1466 return Error::success(); 1467 1468 PythonDictionary globals(PyInitialValue::Empty); 1469 auto builtins = PythonModule::BuiltinsModule(); 1470 if (Error error = globals.SetItem("__builtins__", builtins)) 1471 return error; 1472 PyObject *o = 1473 PyRun_String(script, Py_file_input, globals.get(), globals.get()); 1474 if (!o) 1475 return exception(); 1476 Take<PythonObject>(o); 1477 auto f = As<PythonCallable>(globals.GetItem("main")); 1478 if (!f) 1479 return f.takeError(); 1480 function = std::move(f.get()); 1481 1482 return Error::success(); 1483 } 1484 1485 llvm::Expected<PythonObject> runStringOneLine(const llvm::Twine & string,const PythonDictionary & globals,const PythonDictionary & locals)1486 python::runStringOneLine(const llvm::Twine &string, 1487 const PythonDictionary &globals, 1488 const PythonDictionary &locals) { 1489 if (!globals.IsValid() || !locals.IsValid()) 1490 return nullDeref(); 1491 1492 PyObject *code = 1493 Py_CompileString(NullTerminated(string), "<string>", Py_eval_input); 1494 if (!code) { 1495 PyErr_Clear(); 1496 code = 1497 Py_CompileString(NullTerminated(string), "<string>", Py_single_input); 1498 } 1499 if (!code) 1500 return exception(); 1501 auto code_ref = Take<PythonObject>(code); 1502 1503 PyObject *result = PyEval_EvalCode(code, globals.get(), locals.get()); 1504 1505 if (!result) 1506 return exception(); 1507 1508 return Take<PythonObject>(result); 1509 } 1510 1511 llvm::Expected<PythonObject> runStringMultiLine(const llvm::Twine & string,const PythonDictionary & globals,const PythonDictionary & locals)1512 python::runStringMultiLine(const llvm::Twine &string, 1513 const PythonDictionary &globals, 1514 const PythonDictionary &locals) { 1515 if (!globals.IsValid() || !locals.IsValid()) 1516 return nullDeref(); 1517 PyObject *result = PyRun_String(NullTerminated(string), Py_file_input, 1518 globals.get(), locals.get()); 1519 if (!result) 1520 return exception(); 1521 return Take<PythonObject>(result); 1522 } 1523 1524 #endif 1525