Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add DotNet and NuGet package managers including tests #1303

Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
---
project:
id: "nuget::subProjectTest:"
purl: "pkg://nuget//subProjectTest@"
definition_file_path: "analyzer/src/funTest/assets/projects/synthetic/dotnet/subProjectTest/test.csproj"
declared_licenses: []
declared_licenses_processed: {}
vcs:
type: "Git"
url: "https://github.com/heremaps/oss-review-toolkit.git"
revision: "<REPLACE_REVISION>"
path: "<REPLACE_PATH>"
vcs_processed:
type: "git"
url: "<REPLACE_URL>"
revision: "<REPLACE_REVISION>"
path: "<REPLACE_PATH>"
homepage_url: ""
scopes:
- name: "dependencies"
dependencies:
- id: "nuget::WebGrease:1.5.2"
dependencies:
- id: "nuget::Antlr:3.4.1.9004"
- id: "nuget::Newtonsoft.Json:5.0.4"
- id: "nuget::jQuery:3.3.1"
packages:
- package:
id: "nuget::Antlr:3.4.1.9004"
purl: "pkg://nuget//Antlr@3.4.1.9004"
declared_licenses: []
declared_licenses_processed: {}
description: "ANother Tool for Language Recognition, is a language tool that provides\
\ a framework for constructing recognizers, interpreters, compilers, and translators\
\ from grammatical descriptions containing actions in a variety of target languages."
homepage_url: ""
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/antlr/3.4.1.9004/antlr.3.4.1.9004.nupkg"
hash: "t4RqqB/yvSHU8okrySS1L2KaaPsAOCDM8NPbiOy8OJw/oiNIjjUzS5igIVJds0m5k5AmYyGWV9jBVpFQcq830w=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
- package:
id: "nuget::Newtonsoft.Json:5.0.4"
purl: "pkg://nuget//Newtonsoft.Json@5.0.4"
declared_licenses:
- "http://json.codeplex.com/license"
declared_licenses_processed:
unmapped:
- "http://json.codeplex.com/license"
description: "Json.NET is a popular high-performance JSON framework for .NET"
homepage_url: "http://james.newtonking.com/projects/json-net.aspx"
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/5.0.4/newtonsoft.json.5.0.4.nupkg"
hash: "dg2TSL5YEyhbitOHlaKlr+eIvN2rT3GQ7VgcbqUkwNZ6fdW91ThPGX47WljAcKtb23BEk8yOCtY6s191O21hvw=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
- package:
id: "nuget::WebGrease:1.5.2"
purl: "pkg://nuget//WebGrease@1.5.2"
declared_licenses:
- "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm"
declared_licenses_processed:
unmapped:
- "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm"
description: "Web Grease is a suite of tools for optimizing javascript, css files\
\ and images."
homepage_url: ""
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg"
hash: "RT+no0g7Qk28J/XVwHgn5lJ/qm4hyWFp7wt0I71Uw0j3SfTfOmNqPMxfedHanAjtf3dB2/0KjQ3fQkvVewsvsA=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
- package:
id: "nuget::jQuery:3.3.1"
purl: "pkg://nuget//jQuery@3.3.1"
declared_licenses:
- "http://jquery.org/license"
declared_licenses_processed:
unmapped:
- "http://jquery.org/license"
description: "jQuery is a new kind of JavaScript Library.\n jQuery is a\
\ fast and concise JavaScript Library that simplifies HTML document traversing,\
\ event handling, animating, and Ajax interactions for rapid web development.\
\ jQuery is designed to change the way that you write JavaScript.\n NOTE:\
\ This package is maintained on behalf of the library owners by the NuGet Community\
\ Packages project at http://nugetpackages.codeplex.com/"
homepage_url: "http://jquery.com/"
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/jquery/3.3.1/jquery.3.3.1.nupkg"
hash: "Emsts1D6E2AgHVBm4cTD2dxoRyWVG+3zK1x8mYbf+k6aAArPTXHR2Ip++o1utlNbmYIzaPFnhGw3YDexp8PjlQ=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<PackageReference Include="jQuery" Version="1.3.2"/>
<PackageReference Include="WebGrease" Version="1.5.2"/>
</ItemGroup>

<PropertyGroup>
<TargetFramework>test</TargetFramework>
<RootNamespace>is</RootNamespace>
<AspNetCoreHostingModel>ignored</AspNetCoreHostingModel>
</PropertyGroup>

<ItemGroup>
<Content Remove="compilerconfig.json" />
<Content Remove="wwwroot\css\x.css" />
<Content Remove="wwwroot\css\x.min.css" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
---
project:
id: "nuget::nuget:"
purl: "pkg://nuget//nuget@"
definition_file_path: "analyzer/src/funTest/assets/projects/synthetic/nuget/packages.config"
declared_licenses: []
declared_licenses_processed: {}
vcs:
type: "Git"
url: "https://github.com/heremaps/oss-review-toolkit.git"
revision: "<REPLACE_REVISION>"
path: "<REPLACE_PATH>"
vcs_processed:
type: "git"
url: "<REPLACE_URL>"
revision: "<REPLACE_REVISION>"
path: "<REPLACE_PATH>"
homepage_url: ""
scopes:
- name: "dependencies"
dependencies:
- id: "nuget::WebGrease:1.5.2"
dependencies:
- id: "nuget::Antlr:3.4.1.9004"
- id: "nuget::Newtonsoft.Json:5.0.4"
- id: "nuget::jQuery:3.3.1"
packages:
- package:
id: "nuget::Antlr:3.4.1.9004"
purl: "pkg://nuget//Antlr@3.4.1.9004"
declared_licenses: []
declared_licenses_processed: {}
description: "ANother Tool for Language Recognition, is a language tool that provides\
\ a framework for constructing recognizers, interpreters, compilers, and translators\
\ from grammatical descriptions containing actions in a variety of target languages."
homepage_url: ""
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/antlr/3.4.1.9004/antlr.3.4.1.9004.nupkg"
hash: "t4RqqB/yvSHU8okrySS1L2KaaPsAOCDM8NPbiOy8OJw/oiNIjjUzS5igIVJds0m5k5AmYyGWV9jBVpFQcq830w=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
- package:
id: "nuget::Newtonsoft.Json:5.0.4"
purl: "pkg://nuget//Newtonsoft.Json@5.0.4"
declared_licenses:
- "http://json.codeplex.com/license"
declared_licenses_processed:
unmapped:
- "http://json.codeplex.com/license"
description: "Json.NET is a popular high-performance JSON framework for .NET"
homepage_url: "http://james.newtonking.com/projects/json-net.aspx"
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/5.0.4/newtonsoft.json.5.0.4.nupkg"
hash: "dg2TSL5YEyhbitOHlaKlr+eIvN2rT3GQ7VgcbqUkwNZ6fdW91ThPGX47WljAcKtb23BEk8yOCtY6s191O21hvw=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
- package:
id: "nuget::WebGrease:1.5.2"
purl: "pkg://nuget//WebGrease@1.5.2"
declared_licenses:
- "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm"
declared_licenses_processed:
unmapped:
- "http://www.microsoft.com/web/webpi/eula/msn_webgrease_eula.htm"
description: "Web Grease is a suite of tools for optimizing javascript, css files\
\ and images."
homepage_url: ""
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/webgrease/1.5.2/webgrease.1.5.2.nupkg"
hash: "RT+no0g7Qk28J/XVwHgn5lJ/qm4hyWFp7wt0I71Uw0j3SfTfOmNqPMxfedHanAjtf3dB2/0KjQ3fQkvVewsvsA=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
- package:
id: "nuget::jQuery:3.3.1"
purl: "pkg://nuget//jQuery@3.3.1"
declared_licenses:
- "http://jquery.org/license"
declared_licenses_processed:
unmapped:
- "http://jquery.org/license"
description: "jQuery is a new kind of JavaScript Library.\n jQuery is a\
\ fast and concise JavaScript Library that simplifies HTML document traversing,\
\ event handling, animating, and Ajax interactions for rapid web development.\
\ jQuery is designed to change the way that you write JavaScript.\n NOTE:\
\ This package is maintained on behalf of the library owners by the NuGet Community\
\ Packages project at http://nugetpackages.codeplex.com/"
homepage_url: "http://jquery.com/"
binary_artifact:
url: "https://api.nuget.org/v3-flatcontainer/jquery/3.3.1/jquery.3.3.1.nupkg"
hash: "Emsts1D6E2AgHVBm4cTD2dxoRyWVG+3zK1x8mYbf+k6aAArPTXHR2Ip++o1utlNbmYIzaPFnhGw3YDexp8PjlQ=="
hash_algorithm: "SHA-512"
source_artifact:
url: ""
hash: ""
hash_algorithm: ""
vcs:
type: ""
url: ""
revision: ""
path: ""
vcs_processed:
type: ""
url: ""
revision: ""
path: ""
curations: []
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="jQuery" version="1.3.2" targetFramework="net462" />
<package id="WebGrease" version="1.5.2" targetFramework="net462" />
</packages>
129 changes: 129 additions & 0 deletions analyzer/src/funTest/kotlin/DotNetSupportTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 Bosch Software Innovations GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.analyzer

import com.here.ort.model.Identifier
import com.here.ort.model.OrtIssue
import com.here.ort.model.PackageReference
import com.here.ort.model.Scope

import io.kotlintest.shouldBe
import io.kotlintest.specs.StringSpec

import java.io.File

class DotNetSupportTest : StringSpec() {
private val projectDir = File("src/funTest/assets/projects/synthetic/dotnet")

init {
"non-existing version is mapped to most recent version" {
val testPackage = Pair("jQuery", "1.3.2")
val dotNetSupport = DotNetSupport(mapOf(testPackage), projectDir)
val resultScope = Scope(
"dependencies", sortedSetOf(
PackageReference(
Identifier(
type = "nuget",
namespace = "",
name = "jQuery",
version = "3.3.1"
)
)
)
)

dotNetSupport.scope.toString() shouldBe resultScope.toString()

val resultErrors = listOf<OrtIssue>()

errorsToStringWithoutTimestamp(dotNetSupport.errors) shouldBe errorsToStringWithoutTimestamp(resultErrors)
}

"non-existing project gets registered as error and is not added to scope" {
val testPackage = Pair("trifj", "2.0.0")
val testPackage2 = Pair("tffrifj", "2.0.0")
val dotNetSupport = DotNetSupport(mapOf(testPackage, testPackage2), projectDir)
val resultScope = Scope("dependencies", sortedSetOf())

dotNetSupport.scope.toString() shouldBe resultScope.toString()

val resultErrors = listOf(
OrtIssue(
source = "nuget-API does not provide package",
message = "${testPackage.first}:${testPackage.second} can not be found on Nugets RestAPI "
),
OrtIssue(
source = "nuget-API does not provide package",
message = "${testPackage2.first}:${testPackage2.second} " +
"can not be found on Nugets RestAPI "
)
)

errorsToStringWithoutTimestamp(dotNetSupport.errors) shouldBe errorsToStringWithoutTimestamp(resultErrors)
}

"dependencies are detected correctly" {
val testPackage = Pair("WebGrease", "1.5.2")
val dotNetSupport = DotNetSupport(mapOf(testPackage), projectDir)
val resultScope = Scope(
"dependencies", sortedSetOf(
PackageReference(
Identifier(
type = "nuget",
namespace = "",
name = "WebGrease",
version = "1.5.2"
),
dependencies = sortedSetOf(
PackageReference(
Identifier(
type = "nuget",
namespace = "",
name = "Antlr",
version = "3.4.1.9004"
)
),
PackageReference(
Identifier(
type = "nuget",
namespace = "",
name = "Newtonsoft.Json",
version = "5.0.4"
)
)
)
)
)
)

dotNetSupport.scope.toString() shouldBe resultScope.toString()
}
}

private fun errorsToStringWithoutTimestamp(errors: List<OrtIssue>): String {
var errorResult = ""
errors.forEach { issue: OrtIssue ->
if (errorResult != "") errorResult += ", "
errorResult += issue.toString().split("[ERROR]").last()
}
return "[$errorResult]"
}
}
77 changes: 77 additions & 0 deletions analyzer/src/funTest/kotlin/DotNetTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 Bosch Software Innovations GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.analyzer

import com.here.ort.analyzer.managers.DotNet
import com.here.ort.downloader.VersionControlSystem
import com.here.ort.model.yamlMapper
import com.here.ort.utils.normalizeVcsUrl
import com.here.ort.utils.test.DEFAULT_ANALYZER_CONFIGURATION
import com.here.ort.utils.test.DEFAULT_REPOSITORY_CONFIGURATION
import com.here.ort.utils.test.USER_DIR
import com.here.ort.utils.test.patchExpectedResult

import io.kotlintest.matchers.beEmpty
import io.kotlintest.should
import io.kotlintest.shouldBe
import io.kotlintest.shouldNotBe
import io.kotlintest.specs.StringSpec

import com.fasterxml.jackson.dataformat.xml.XmlMapper
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule

import java.io.File

class DotNetTest : StringSpec() {
private val projectDir = File("src/funTest/assets/projects/synthetic/dotnet")
private val vcsDir = VersionControlSystem.forDirectory(projectDir.absoluteFile)!!
private val vcsUrl = vcsDir.getRemoteUrl()
private val vcsRevision = vcsDir.getRevision()
private val packageFile = File(projectDir, "subProjectTest/test.csproj")

init {
"Project dependencies are detected correctly" {
val vcsPath = vcsDir.getPathToRoot(projectDir)
val expectedResult = patchExpectedResult(File(projectDir.parentFile,
"dotnet-expected-output.yml"),
definitionFilePath = "$vcsPath/subProjectTest/test.csproj",
path = "$vcsPath/subProjectTest",
revision = vcsRevision,
url = normalizeVcsUrl(vcsUrl))
val result = DotNet("DotNet", DEFAULT_ANALYZER_CONFIGURATION, DEFAULT_REPOSITORY_CONFIGURATION)
.resolveDependencies(USER_DIR, listOf(packageFile))[packageFile]

result shouldNotBe null
result!!.errors should beEmpty()
yamlMapper.writeValueAsString(result) shouldBe expectedResult
}

"Definition File is correctly mapped" {
val mapper = XmlMapper().registerKotlinModule()
val result = mapper.readValue<List<DotNet.Companion.ItemGroup>>(packageFile)

result shouldNotBe null
result.size shouldBe 4
result[1].packageReference?.size shouldBe 2
}
}
}
77 changes: 77 additions & 0 deletions analyzer/src/funTest/kotlin/NugetTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 Bosch Software Innovations GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.analyzer

import com.fasterxml.jackson.dataformat.xml.XmlMapper
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule

import com.here.ort.analyzer.managers.Nuget
import com.here.ort.downloader.VersionControlSystem
import com.here.ort.model.yamlMapper
import com.here.ort.utils.normalizeVcsUrl
import com.here.ort.utils.test.DEFAULT_ANALYZER_CONFIGURATION
import com.here.ort.utils.test.DEFAULT_REPOSITORY_CONFIGURATION
import com.here.ort.utils.test.USER_DIR
import com.here.ort.utils.test.patchExpectedResult

import io.kotlintest.matchers.beEmpty
import io.kotlintest.should
import io.kotlintest.shouldBe
import io.kotlintest.shouldNotBe
import io.kotlintest.specs.StringSpec

import java.io.File

class NugetTest : StringSpec() {
private val projectDir = File("src/funTest/assets/projects/synthetic/nuget")
private val vcsDir = VersionControlSystem.forDirectory(projectDir.absoluteFile)!!
private val vcsUrl = vcsDir.getRemoteUrl()
private val vcsRevision = vcsDir.getRevision()
private val packageFile = File(projectDir, "packages.config")

init {
"Project dependencies are detected correctly" {
val vcsPath = vcsDir.getPathToRoot(projectDir)
val expectedResult = patchExpectedResult(File(projectDir.parentFile,
"nuget-expected-output.yml"),
definitionFilePath = "$vcsPath/packages.config",
path = vcsPath,
revision = vcsRevision,
url = normalizeVcsUrl(vcsUrl))
val result = Nuget("Nuget", DEFAULT_ANALYZER_CONFIGURATION, DEFAULT_REPOSITORY_CONFIGURATION)
.resolveDependencies(USER_DIR, listOf(packageFile))[packageFile]

result shouldNotBe null
result!!.errors should beEmpty()
yamlMapper.writeValueAsString(result) shouldBe expectedResult
}

"Definition File is correctly mapped" {
val mapper = XmlMapper().registerKotlinModule()
val result: Nuget.Companion.Packages = mapper.readValue(packageFile)

result shouldNotBe null
result.packages shouldNotBe null
result.packages?.size shouldBe 2
}
}
}
334 changes: 334 additions & 0 deletions analyzer/src/main/kotlin/DotNetSupport.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,334 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 Bosch Software Innovations GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.analyzer

import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper

import com.here.ort.model.HashAlgorithm
import com.here.ort.model.Identifier
import com.here.ort.model.OrtIssue
import com.here.ort.model.Package
import com.here.ort.model.PackageReference
import com.here.ort.model.RemoteArtifact
import com.here.ort.model.Scope
import com.here.ort.model.VcsInfo
import com.here.ort.model.xmlMapper
import com.here.ort.utils.OkHttpClientHelper
import com.here.ort.utils.textValueOrEmpty

import okhttp3.Request

import java.io.File
import java.io.IOException
import java.net.HttpURLConnection

class DotNetSupport(
packageReferencesMap: Map<String, String>,
val workingDir: File
) {
companion object {
private const val SCOPE_NAME = "dependencies"
private const val PROVIDER_NAME = "nuget"

private fun extractRepositoryType(node: JsonNode) =
node["repository"]?.get("type")?.textValue()
?: ""

private fun extractRepositoryUrl(node: JsonNode) =
node["repository"]?.get("url")?.textValue()
?: node["projectURL"]?.textValue()
?: ""

private fun extractRepositoryRevision(node: JsonNode): String =
node["repository"]?.get("commit")?.textValue()
?: ""

private fun extractRepositoryPath(node: JsonNode): String =
node["repository"]?.get("branch")?.textValue()
?: ""

private fun extractVcsInfo(node: JsonNode) =
VcsInfo(
type = extractRepositoryType(node),
url = extractRepositoryUrl(node),
revision = extractRepositoryRevision(node),
path = extractRepositoryPath(node)
)

private fun extractPackageId(node: JsonNode) = Identifier(
type = PROVIDER_NAME,
namespace = "",
name = node["id"]?.textValue() ?: "",
version = node["version"]?.textValue() ?: ""
)


private fun extractDeclaredLicenses(node: JsonNode) = sortedSetOf<String>().apply {
val license = node["license"]?.textValue() ?: node["licenseUrl"]?.textValue() ?: ""
// most nuget packages only provide a licenseUrl,
// which counts as declared license, will be changed when scanner runs
if (license.isNotEmpty()) {
add(license)
}
}

private fun extractRemoteArtifact(node: JsonNode, nupkgUrl: String) = RemoteArtifact(
url = nupkgUrl,
hash = node["packageHash"]?.textValue() ?: "",
hashAlgorithm = HashAlgorithm.fromString(
node["packageHashAlgorithm"]?.textValue() ?: ""
)
)

private fun getCatalogURL(registrationNode: JsonNode): String =
registrationNode["catalogEntry"]?.textValue() ?: ""

private fun getNuspecURL(registrationNode: JsonNode): String =
registrationNode["packageContent"]?.textValue() ?: ""

private fun extractVersion(range: String): String {
if (range.isEmpty()) return ""
val rangeReplaces = range.replace("[", "")
.replace(" ", "")
.replace(")", "")
return rangeReplaces.split(",").elementAt(0)
}

}

var packages = mutableMapOf<String, Package>()
var errors = mutableListOf<OrtIssue>()
var scope = Scope(SCOPE_NAME, sortedSetOf())
// consists of map key: id, version and pair map entry: nupkg url as string and catalogentry as jsonnode
private val packageReferencesAlreadyFound = mutableMapOf<Pair<String, String>, Pair<String, JsonNode>>()

init {
packageReferencesMap.forEach { name, version ->
val scopeDependency = getPackageReferenceFromRestAPI(name, version)
if (scopeDependency != null) scope.dependencies += scopeDependency
}

scope.collectDependencies().forEach { packageReference ->
val pkg = packageReferenceToPackage(packageReference)

if (pkg != Package.EMPTY) {
packages["${pkg.id.name}:${pkg.id.version}"] = pkg
}
}
}

private fun packageReferenceToPackage(packageReference: PackageReference): Package {
return jsonNodeToPackage(getPackageReferenceJsonContent(packageReference))
}

private fun getPackageReferenceJsonContent(packageReference: PackageReference): Pair<String, JsonNode> {
if (packageReferencesAlreadyFound.containsKey(
Pair(packageReference.id.name, packageReference.id.version)
)) {
return packageReferencesAlreadyFound[Pair(
packageReference.id.name,
packageReference.id.version)]!!
}
return try {
val informationUrl = getInformationURL(packageReference.id.name,
packageReference.id.version) ?: throw Exception()
Pair(informationUrl.first, jacksonObjectMapper().readTree(informationUrl.second.requestFromNugetAPI()))
} catch (e: Exception) {
Pair("", xmlMapper.readTree(""))
}
}

private fun jsonNodeToPackage(packageContent: Pair<String, JsonNode>): Package {
val jsonCatalogNode = packageContent.second
val jsonNuspecNode = try {
xmlMapper.readTree(packageContent.first.replace(
"${jsonCatalogNode["version"].textValueOrEmpty()}.nupkg",
"nuspec"
).requestFromNugetAPI())
} catch (e: Exception) {
xmlMapper.readTree("")
}
if (jsonCatalogNode["id"]?.textValue() == null) return Package.EMPTY

val vcsInfo = extractVcsInfo(jsonNuspecNode["metadata"] ?: xmlMapper.readTree(""))

return Package(
id = extractPackageId(jsonCatalogNode),
declaredLicenses = extractDeclaredLicenses(jsonCatalogNode),
description = jsonCatalogNode["description"]?.textValue() ?: "",
homepageUrl = jsonCatalogNode["projectUrl"]?.textValue() ?: "",
binaryArtifact = extractRemoteArtifact(jsonCatalogNode, packageContent.first),
sourceArtifact = RemoteArtifact.EMPTY,
vcs = vcsInfo,
vcsProcessed = vcsInfo.normalize()
)
}

private fun getPackageReferenceFromRestAPI(packageID: String, version: String): PackageReference? {
val packageJsonNode = preparePackageReference(packageID, version)

if (packageJsonNode == null) {
errors.add(OrtIssue(
source = "nuget-API does not provide package",
message = "$packageID:$version can not be found on Nugets RestAPI "
))
return null
}

val packageReference = PackageReference(
Identifier(
type = PROVIDER_NAME,
namespace = "",
name = if (packageID == packageJsonNode["id"]?.textValue()) {
packageID
} else {
packageJsonNode["id"]?.textValue() ?: ""
},
version = packageJsonNode["version"]?.textValue() ?: ""
))

val dependenciesIterator = packageJsonNode["dependencyGroups"]?.elements()

dependenciesIterator?.forEach {
val dependencyIterator = it["dependencies"]?.elements()

if (dependencyIterator != null) while (dependencyIterator.hasNext()) {
val node = dependencyIterator.next()
val nodeAsPair = Pair(
node["id"].textValueOrEmpty(),
extractVersion(node["range"].textValueOrEmpty()))
if (!hasPackageReferenceAlready(nodeAsPair)) {
val subPackageRef = getPackageReferenceFromRestAPI(
nodeAsPair.first,
nodeAsPair.second
)
if (subPackageRef != null) packageReference.dependencies += subPackageRef
}
}
}

return packageReference
}

private fun preparePackageReference(packageID: String, version: String): JsonNode? {
val (first, second) = getInformationURL(packageID, version) ?: return null
val packageJsonNode = try {
jacksonObjectMapper().readTree(
second.requestFromNugetAPI())
?: throw Exception("No configuration URL could be fetched")
} catch (e: Exception) {
xmlMapper.readTree("")
}

packageReferencesAlreadyFound[Pair(first = packageID, second = version)] =
Pair(first, packageJsonNode)

return packageJsonNode
}

private fun hasPackageReferenceAlready(nodeAsPair: Pair<String, String>): Boolean {
return packageReferencesAlreadyFound.containsKey(nodeAsPair)
}

private fun getInformationURL(packageID: String, version: String): Pair<String, String>? {
val registrationInfo: String? = try {
"https://api.nuget.org/v3/registration3/$packageID/$version.json".requestFromNugetAPI()
} catch (e: IOException) {
try {
getIdUrl(packageID, version).requestFromNugetAPI()
} catch (e: IOException) {
""
}
}

val node = jacksonObjectMapper().readTree(
registrationInfo ?: ""
)

return if (node != null) {
Pair(getNuspecURL(node), getCatalogURL(node))
} else {
null
}
}

private fun getIdUrl(packageID: String, version: String): String {
val node = getCreateSearchRestAPIURL(packageID)

return getRightVersionUrl(node["data"]?.elements(), packageID, version)
?: getFirstMatchingIdUrl(node["data"]?.elements(), packageID) ?: ""
}

private fun getRightVersionUrl(dataIterator: Iterator<JsonNode>?,
packageID: String, version: String): String? {
while (dataIterator != null && dataIterator.hasNext()) {
val packageNode = dataIterator.next()
if (packageNode["id"].textValueOrEmpty() == packageID) {
packageNode["versions"].elements().forEach {
if (it["version"].textValueOrEmpty() == version)
return it["@id"].textValueOrEmpty()
else if (!dataIterator.hasNext() && version == "latest")
return it["@id"].textValueOrEmpty()
}
}
}

return null
}

private fun getFirstMatchingIdUrl(dataIterator: Iterator<JsonNode>?, packageID: String): String? {
while (dataIterator != null && dataIterator.hasNext()) {
val packageNode = dataIterator.next()
if (packageNode["id"].textValueOrEmpty() == packageID) {
return packageNode["versions"]?.last()?.get("@id").textValueOrEmpty()
}
}

return null
}

private fun getCreateSearchRestAPIURL(packageID: String): JsonNode {
return jacksonObjectMapper().readTree(
"https://api-v2v3search-0.nuget.org/query?q=\"$packageID\"&prerelease=false".requestFromNugetAPI())
}

private fun String.requestFromNugetAPI(): String {
if (isNullOrEmpty()) {
throw IOException("GET with URL $this could not be resolved")
}

val pkgRequest = Request.Builder()
.get()
.url(this)
.build()

OkHttpClientHelper.execute(HTTP_CACHE_PATH, pkgRequest).use { response ->
val body = response.body()?.string()?.trim()

if (response.code() != HttpURLConnection.HTTP_OK || body.isNullOrEmpty()) {
throw IOException("GET with URL $this could not be resolved")
}

return body
}
}
}
117 changes: 117 additions & 0 deletions analyzer/src/main/kotlin/managers/DotNet.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 Bosch Software Innovations GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.analyzer.managers

import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.dataformat.xml.XmlMapper
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule

import com.here.ort.analyzer.AbstractPackageManagerFactory
import com.here.ort.analyzer.DotNetSupport
import com.here.ort.analyzer.PackageManager
import com.here.ort.downloader.VersionControlSystem
import com.here.ort.model.Identifier
import com.here.ort.model.Project
import com.here.ort.model.ProjectAnalyzerResult
import com.here.ort.model.config.AnalyzerConfiguration
import com.here.ort.model.config.RepositoryConfiguration

import java.io.File

/**
* Dotnet package manager
*/
open class DotNet(name: String, analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) :
PackageManager(name, analyzerConfig, repoConfig) {
companion object {
fun mapPackageReferences(workingDir: File): Map<String, String> {
val map = mutableMapOf<String, String>()
val mapper = XmlMapper().registerKotlinModule()
val mappedFile: List<ItemGroup> = mapper.readValue(workingDir)

mappedFile.forEach { itemGroup ->
itemGroup.packageReference?.forEach {
if (!it.Include.isNullOrEmpty()) {
map[it.Include] = it.Version ?: " "
}
}
}

return map
}

@JsonIgnoreProperties(ignoreUnknown = true)
data class ItemGroup(
@JsonProperty(value = "PackageReference")
@JacksonXmlElementWrapper(useWrapping = false)
val packageReference: List<PackageReference>?
)

@JsonIgnoreProperties(ignoreUnknown = true)
data class PackageReference(
@JsonProperty(value = "Include")
@JacksonXmlProperty(isAttribute = true)
val Include: String?,
@JsonProperty(value = "Version")
@JacksonXmlProperty(isAttribute = true)
val Version: String?
)
}

class Factory : AbstractPackageManagerFactory<DotNet>("DotNet") {
override val globsForDefinitionFiles = listOf("*.csproj")

override fun create(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) =
DotNet(managerName, analyzerConfig, repoConfig)
}

override fun resolveDependencies(definitionFile: File): ProjectAnalyzerResult? {
val workingDir = definitionFile.parentFile
val dotnet = DotNetSupport(mapPackageReferences(definitionFile), workingDir)
val vcsInfo = VersionControlSystem.getPathInfo(workingDir)

val project = Project(
id = Identifier(
type = "nuget",
namespace = "",
name = workingDir.name,
version = ""
),
definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path,
declaredLicenses = sortedSetOf(),
vcs = vcsInfo,
vcsProcessed = vcsInfo.normalize(),
homepageUrl = "",
scopes = sortedSetOf(dotnet.scope)
)

return ProjectAnalyzerResult(
project,
packages = dotnet.packages.values.map { it.toCuratedPackage() }.toSortedSet(),
errors = dotnet.errors
)
}
}

109 changes: 109 additions & 0 deletions analyzer/src/main/kotlin/managers/Nuget.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Copyright (C) 2017-2019 HERE Europe B.V.
* Copyright (C) 2019 Bosch Software Innovations GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package com.here.ort.analyzer.managers

import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.dataformat.xml.XmlMapper
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty
import com.fasterxml.jackson.module.kotlin.readValue
import com.fasterxml.jackson.module.kotlin.registerKotlinModule

import com.here.ort.analyzer.AbstractPackageManagerFactory
import com.here.ort.analyzer.DotNetSupport
import com.here.ort.analyzer.PackageManager
import com.here.ort.downloader.VersionControlSystem
import com.here.ort.model.Identifier
import com.here.ort.model.Project
import com.here.ort.model.ProjectAnalyzerResult
import com.here.ort.model.config.AnalyzerConfiguration
import com.here.ort.model.config.RepositoryConfiguration

import java.io.File

/**
* The Nuget package mangager for .NET, see https://www.nuget.org/
*/
open class Nuget(name: String, analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) :
PackageManager(name, analyzerConfig, repoConfig) {
companion object {
fun mapPackageReferences(workingDir: File): Map<String, String> {
val map = mutableMapOf<String, String>()
val mapper = XmlMapper().registerKotlinModule()
val mappedFile: Packages = mapper.readValue(workingDir)

mappedFile.packages?.forEach {
map[it.id] = it.version
}

return map
}

@JsonIgnoreProperties(ignoreUnknown = true)
data class Packages(
@JsonProperty(value = "package")
@JacksonXmlElementWrapper(useWrapping = false)
val packages: List<Package>?
)

@JsonIgnoreProperties(ignoreUnknown = true)
data class Package(
@JacksonXmlProperty(isAttribute = true)
val id: String,
@JacksonXmlProperty(isAttribute = true)
val version: String)
}

class Factory : AbstractPackageManagerFactory<Nuget>("Nuget") {
override val globsForDefinitionFiles = listOf("packages.config")

override fun create(analyzerConfig: AnalyzerConfiguration, repoConfig: RepositoryConfiguration) =
Nuget(managerName, analyzerConfig, repoConfig)
}

override fun resolveDependencies(definitionFile: File): ProjectAnalyzerResult? {
val workingDir = definitionFile.parentFile
val nuget = DotNetSupport(mapPackageReferences(definitionFile), workingDir)
val vcsInfo = VersionControlSystem.getPathInfo(workingDir)

val project = Project(
id = Identifier(
type = "nuget",
namespace = "",
name = workingDir.name,
version = ""
),
definitionFilePath = VersionControlSystem.getPathInfo(definitionFile).path,
declaredLicenses = sortedSetOf(),
vcs = vcsInfo,
vcsProcessed = vcsInfo.normalize(),
homepageUrl = "",
scopes = sortedSetOf(nuget.scope)
)

return ProjectAnalyzerResult(
project = project,
packages = nuget.packages.values.map { it.toCuratedPackage() }.toSortedSet(),
errors = nuget.errors
)
}
}
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
com.here.ort.analyzer.managers.Bower$Factory
com.here.ort.analyzer.managers.Bundler$Factory
com.here.ort.analyzer.managers.DotNet$Factory
com.here.ort.analyzer.managers.GoDep$Factory
com.here.ort.analyzer.managers.Gradle$Factory
com.here.ort.analyzer.managers.Maven$Factory
com.here.ort.analyzer.managers.NPM$Factory
com.here.ort.analyzer.managers.Nuget$Factory
com.here.ort.analyzer.managers.PhpComposer$Factory
com.here.ort.analyzer.managers.PIP$Factory
com.here.ort.analyzer.managers.SBT$Factory
2 changes: 2 additions & 0 deletions analyzer/src/test/kotlin/PackageManagerTest.kt
Original file line number Diff line number Diff line change
@@ -46,10 +46,12 @@ class PackageManagerTest : WordSpec({

managedFilesByName["Bower"] shouldBe listOf(File(projectDir, "bower.json"))
managedFilesByName["Bundler"] shouldBe listOf(File(projectDir, "Gemfile"))
managedFilesByName["DotNet"] shouldBe listOf(File(projectDir, "test.csproj"))
managedFilesByName["GoDep"] shouldBe listOf(File(projectDir, "Gopkg.toml"))
managedFilesByName["Gradle"] shouldBe listOf(File(projectDir, "build.gradle"))
managedFilesByName["Maven"] shouldBe listOf(File(projectDir, "pom.xml"))
managedFilesByName["NPM"] shouldBe listOf(File(projectDir, "package.json"))
managedFilesByName["Nuget"] shouldBe listOf(File(projectDir, "packages.config"))
managedFilesByName["PhpComposer"] shouldBe listOf(File(projectDir, "composer.json"))
managedFilesByName["PIP"] shouldBe listOf(File(projectDir, "setup.py"))
managedFilesByName["SBT"] shouldBe listOf(File(projectDir, "build.sbt"))